diff --git a/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md b/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md index e4daa97024..90cf08eeb3 100644 --- a/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md +++ b/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md @@ -93,10 +93,14 @@ Use `view: {kind: "environment-state"}` with the manifest's `template_rules` to extract the data those reads observed instead: entity types and identity fields, entities with recorded values, documents, and per-tool render plans checked against the recordings. The state comes only from reads before the -first related write; types, identity and list membership are inferences. +first related write; types, identity and list membership are inferences. The +view also gives each state-changing tool a write plan, served only when a later +recorded read confirms its effect and none contradicts it. `rea environment-serve manifest.json --evidence capture.json` derives that -state at startup, answers read tools from it and serves `data/reset`, -`data/add` and `data/get` on `/agentenv`; write tools are not served. It -replays the capture instead with `--replay-only` or when the state is -unavailable (not derivable, or a catalog over the discovery budget), and its -stopped document's `state` says which applied. +state at startup, answers read tools from it, serves the write tools whose plan +is served, each applying its effect to that state, and serves `data/reset`, +`data/add` and `data/get` on `/agentenv`; other write tools are not served, and +each plan states why in its `reason`. It replays the capture instead with +`--replay-only` or when the state is unavailable (not derivable, or a catalog +over the discovery budget), and its stopped document's `state` says which +applied. diff --git a/docs/agent-environments.md b/docs/agent-environments.md index c2398fc3da..431566f5f0 100644 --- a/docs/agent-environments.md +++ b/docs/agent-environments.md @@ -5,7 +5,8 @@ environment: an MCP server whose tools answer from what one retained web network capture observed, plus the environment card the agentenv framework reads to discover it. By default the server derives the [state the capture observed](#the-state-a-capture-observed) at startup and -answers read tools from it; `--replay-only` replays the capture's records +answers read tools from it, and the write tools whose effect the capture +confirmed by changing it; `--replay-only` replays the capture's records instead. Nothing is fetched from the recorded application; every answer comes from retained Evidence. `rea environment-export` writes the same pair as a bundle folder `agent-env run` builds and runs; see @@ -271,11 +272,11 @@ document, with the same signal exit status. Three routes are served: -| Route | Answer | -| --------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `POST /mcp` | The MCP server: from the state, one tool per read tool, answering from the state where it backs them; in replay, one tool per manifest tool, replaying records | -| `GET /.well-known/agent-env.json` | The environment card | -| `POST /agentenv` | From the state, `data/reset`, `data/add` and `data/get`; in replay, JSON-RPC `-32601 method not found: ` for every method | +| Route | Answer | +| --------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `POST /mcp` | The MCP server: from the state, one tool per read tool, answering from the state where it backs them, and one per write tool whose effect the capture confirmed, applying it to the state; in replay, one tool per manifest tool, replaying records | +| `GET /.well-known/agent-env.json` | The environment card | +| `POST /agentenv` | From the state, `data/reset`, `data/add` and `data/get`; in replay, JSON-RPC `-32601 method not found: ` for every method | Numeric JSON-RPC IDs outside JavaScript's safe range or whose precision would be lost are refused as invalid requests with a null ID. They are never echoed @@ -376,8 +377,11 @@ In replay, `extensions`, `operations` and `children_environments` are fixed to `null`, `[]` and `null`: a replay environment declares no extensions, no data-plane operations and no children. Served [from the state](#serve-from-the-state), the card declares the three data-plane -operations and one intake extension instead, leaves write tools out and -describes each tool as answering from the state where the state backs it. +operations and one intake extension instead, and the catalog holds the read +tools and only the write tools whose effect the capture confirmed: a read tool +is described as answering from the state where the state backs it, and a +served write tool by the effect it applies to the state and the body members it +accepts. ## Serve from the state @@ -400,7 +404,8 @@ A state that is unavailable does not stop the server: it logs a warning naming the error `code` and message before it binds, then serves replay. The state is unavailable when it cannot be derived, such as from a `capture_browser_scenario` capture that selected header values but no response -bodies or over the view's budget, and when its catalog under the served name +bodies or over the view's budget, when `data/get` would write it past the 32 MiB +`data/add` intake limit, and when its catalog under the served name exceeds the [discovery reservation](#what-a-manifest-is). That check runs first, so no state is derived for it, and replay is then admitted exactly as with `--replay-only`. `--replay-only` @@ -417,26 +422,55 @@ member reports which applied: Each read tool answers according to its view in the state: -| View | Answer | -| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `item` | The entity the path arguments name, rendered into the recorded envelope of `envelope_ordinal`. Arguments match as replay matches them: each collection segment equals the held collection's segment as written or after percent-decoding, and the identity is the last argument percent-decoded, then as written; values are never re-split on `/`. Any entity the state holds answers, including one `data/add` loaded; query arguments only decide `replay.exact_match`. An entity the state does not hold, or one lacking a field every recording carried, is a tool error; a body is never invented. | -| `list` | When the arguments name one of the lines the view checked exactly (its latest recordings before the first related write; every path and query value equal to the recording's, and every recorded query parameter supplied), that recorded envelope with its array rendered from the state: every entity of the collection in state order for `membership: "all"`, or the recorded members still held. A member lacking a field every recording carried makes the call a tool error rather than a partial list. | -| `document` | When the arguments name the document's checked line exactly, the state's value for it; a tool error when the state holds none. | -| `recorded` | The recording, as replay answers it. | +| View | Answer | +| ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `item` | The entity the path arguments name, rendered into the recorded envelope of `envelope_ordinal`. Arguments match as replay matches them: each collection segment equals the held collection's segment as written or after percent-decoding, and the identity is the last argument percent-decoded, then as written; values are never re-split on `/`. Any entity the state holds answers, including one `data/add` loaded; query arguments only decide `replay.exact_match`. An entity the state does not hold answers the tool's latest recorded 400, 404 or 410 response with `state.view` `missing`, or is a tool error when none was recorded; one lacking a field every recording carried is a tool error; a body is never invented. | +| `list` | When the arguments name one of the lines the view checked exactly (its latest recordings before the first related write; every path and query value equal to the recording's, and every recorded query parameter supplied), that recorded envelope with its array rendered from the state: every entity of the collection in state order for `membership: "all"`, or the recorded members still held. A call with no query argument to a `membership: "all"` view answers from a checked line of the collection its path arguments name that only pages its list, with `replay.exact_match` false. A member lacking a field every recording carried makes the call a tool error rather than a partial list. | +| `document` | When the arguments name the document's checked line exactly, the state's value for it; a tool error when the state holds none. | +| `recorded` | The recording, as replay answers it. | A list or document call that names no checked line, including a line recorded after a write or with other query values, answers from the recording. Every answer has the replay shape plus `state`: `{"source": "state", "view": …}` naming the entity, collection or document path, or `{"source": "recording", -"reason": …}` with the view's reason, `no-view` or `unrecorded-request`. A +"reason": …}` with the view's reason, `no-view`, `unrecorded-request` or `page-exceeded` (a call naming a page of a whole list that the collection has outgrown). A state answer carries the base record's status, content type and `replay` provenance with `truncated: false`; one whose JSON would exceed 256 KiB is refused as a `resource_constraint` tool error rather than cut. Extraction does not state-back a document whose rendered JSON exceeds 256 KiB; that read stays -recorded (`rendered-oversize`) and replays the captured body. Serving is -read-only: `POST`, `PUT`, `PATCH` and `DELETE` tools, other than query `POST`s, -are not served, so nothing a call does changes the state. Query `POST`s stay -served and answer from their recordings with the reason `query-body`. +recorded (`rendered-oversize`) and replays the captured body. Query `POST`s +stay served and answer from their recordings with the reason `query-body`. + +`POST`, `PUT`, `PATCH` and `DELETE` tools are served only when the state's +[write plan](web-network-captures.md#environment-state) for them is served: +the capture's later reads confirmed the effect the plan models. Other write +tools are not served, so no call claims a change it does not make; each one's +plan states why in its `reason`. A served write changes the live state, which +every MCP session and `data/get` then see: + +| Effect | Call | +| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `create` | Adds an entity to the collection the path names, built from the observed entity with each ruled field taken from the call or generated, at the plan's `insert` position. A plan tied to its exemplar's collection writes only there. | +| `update` | Sets the sent members on the entity the path names, constants for members accepted only with their recorded value, and the clock, increment and slug changes the plan models. | +| `delete` | Removes the entity the path names. | +| `document` | Merges the body into the plan's document object, or replaces it for `PUT`. | + +Its answer has the status and content type of the plan's recorded response, +with the written entity or its identity rendered in, no body, or the recorded +body. A call whose body is not an object, sends a member the capture did not +show stored, sends a fixed member with another value or a member of another +JSON type, or lacks a required member is a tool error naming the accepted +members; so is a rendered answer missing a field the recorded response +carried, and an update or delete of an entity the state does not hold answers +the recorded 400, 404 or 410 of the tool or of an item view of the type, if +any. A refused call leaves the state unchanged. Values a write generates are +deterministic: the clock starts at `write_clock` and advances one second per +served write, sequences take one past both the largest held value and the +largest they generated, and UUIDs and tokens derive from the state Evidence +id, tool, field and write count, skipping held values. `data/reset` and +`data/add` restart the clock, counter and sequences, so the same calls after a +reset give the same state. Answers carry `state: {"source": "state", "view": +"create" | "update" | "delete", …}` or `"document-write"`. The data plane serves three JSON-RPC methods: @@ -470,18 +504,29 @@ names its category in `error.data.reason`: `not-found`, `permission`, downloads, and `cancelled` for a download the server's stop or its caller's departure ended. +Served creates also bound additional collection history since the last reset +or `data/add` to 1 MiB of deterministic accounting: UTF-8 bytes for each new +collection spelling and its possible sequence keys, plus a 64-byte allowance +per collection and potential sequence-key record. The score bounds identifier +payload and a fixed record allowance; it is not a measurement or guarantee of +process memory. Collections +already present in the Evidence or loaded state are part of the state itself; +the store indexes their unique spellings so they remain known after deletion, +but they do not consume this additional-history budget. A create that would +exceed the budget is refused before the state changes. `data/reset` or `data/add` +clears generated history; if the same calls would hit the limit again, use +fewer distinct collection paths. The 1 MiB score ceiling is 1/32 of the +separate 32 MiB state-intake ceiling, reserving a small, explicit allowance +for identifiers retained beyond the loaded state. A loaded-state path that is +later deleted and then introduced again may consume the additional-history +budget when the store must retain it beyond the source entity. + Stopping the server cancels every pending call, and a caller that leaves before its reply cancels its own: a download in progress ends, and calls not yet run are answered `-32000` with `error.data.reason` `cancelled` without running. A cancelled call leaves the state unchanged, and shutdown never waits for a download. -A `file://` state file whose close fails stays owned by the server, and -the `data/add` gets `-32000`. The next `file://` load retries that close first -and, while it still fails, answers `-32000` without reading another file; -inline and `https://` documents still load. Closing the server retries it once -more and fails with the unfinished close rather than dropping the descriptor. - One `/agentenv` response carries at most 64 MiB of replies, twice the 32 MiB a loaded document may hold: room for a `data/get` of a loaded state, while a batch of reads cannot multiply a large state into one body. Replies are @@ -565,14 +610,17 @@ The card declares the operations and what `data/add` accepts: } ``` -The state is a projection of recorded reads. List and document calls answer -from the state only for the lines their views checked, and every recorded view -answers from recordings; item calls answer for any held entity but render only -the envelope one recording carried. Values copied outside entity objects, -derived values and counters other than those that count a list stay as -recorded; no write has an effect. An exported bundle carries no state file: -its server derives the state at startup, as the export does to predict what it -serves. +The state starts as a projection of recorded reads and changes only through +served write tools and the data plane. List and document calls answer from the +state only for the lines their views checked, and every recorded view answers +from recordings; item calls answer for any held entity but render only the +envelope one recording carried. Values copied outside entity objects, derived +values and counters other than those that count a list stay as recorded, and a +served write changes only the one entity or document it writes: side effects +on counters, relations and copies of entities in other collections, reordering +after an update and referential rules on delete are not modelled. An exported +bundle carries no state file: its server derives the state at startup, as the +export does to predict what it serves. ## Export a bundle @@ -640,8 +688,8 @@ name from Python sources the image does not have. `verify.py` is the framework's `async def verify(mcp_url)` shape over its Python MCP client: it lists the tools and checks the count equals the number -the bundle serves (the manifest's read tools from the state, every manifest -tool in replay), calls one tool with no arguments and checks the call did not +the bundle serves (from the state, the manifest's read tools and the write +tools whose effect the capture confirmed; every manifest tool in replay), calls one tool with no arguments and checks the call did not fail and its `status` is the expected one, and returns one `serve` result whose description reports both checks. The tool is the first served `GET` tool without path parameters whose answer to no arguments is not an error, and the @@ -659,8 +707,9 @@ The README written into the bundle records the capture artifact's SHA-256, the capture and manifest Evidence ids, the manifest view digest, the served tool count, the derived state's Evidence id or that no state is served, and the package source. It says what the environment serves: read tools from the state -the capture observed, the `/agentenv` data plane on it and no write tools yet, -replay when the state is unavailable, and `--replay-only` for pure replay; it +the capture observed, the `/agentenv` data plane on it and the write tools +whose effect the capture's later reads confirmed, replay when the state is +unavailable, and `--replay-only` for pure replay; it says which applies to this bundle, with the error code and message when the state is unavailable, and that neither is emulation and served content is untrusted data. It states that the recorded content belongs to the diff --git a/docs/cli.md b/docs/cli.md index 27f9feb215..b263b323d3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -414,7 +414,8 @@ evidence file's `evidence_id`; otherwise the command refuses with `invalid_request` before binding. Before binding it derives the `environment-state` view of the capture with the manifest's `template_rules`, as `inspect-analysis-view` would, and serves from it: read tools whose views -are state-backed answer from a live state, write tools are not served, and +are state-backed answer from a live state, write tools are served only where +the capture's later reads confirmed their effect, and `/agentenv` serves `data/reset`, `data/add` and `data/get`. When the state cannot be derived, or its catalog exceeds the discovery reservation, it logs a warning with the error code and message and serves replay; `--replay-only` serves replay without deriving. In replay every diff --git a/docs/specs/agent-environments/state-writes/plan.md b/docs/specs/agent-environments/state-writes/plan.md new file mode 100644 index 0000000000..e8550238b2 --- /dev/null +++ b/docs/specs/agent-environments/state-writes/plan.md @@ -0,0 +1,45 @@ +# Implementation plan + +The branch builds on the state-serving and coverage branches (merged into it). + +1. **Write plan types and request bodies.** Add the plan, rule and reason types + beside the state rules, and have the state accumulator lend extraction + parsed request bodies under the same budget, admitted like response bodies. +2. **Field rules.** Add a module that explains an observed entity against the + call that wrote it (echo, copy, path, slug, clock, sequence, unique, + template and constant) and that recognises and formats clock values, + UUIDs and tokens. +3. **Effects.** Add one apply module that both the gate and the server use. It + applies a create, update, delete or document write to a writable state + interface from a plan, a call and a value source: the recorded values in + the gate, the store's generators when served. +4. **Gate.** Add a simulation that: + - classifies every write tool; + - builds each plan from its exemplar; + - walks the timeline over a copy of the initial state; + - checks state-answered reads; + - attributes passes and mismatches; + - learns the insert rule; + - records verdicts. + + Extraction runs it after the read gate and returns `writes` and + `write_clock`; the view schema and its unknowns state the write model. + +5. **Serving.** + - The loaded state carries the plans and the clock. + - The store gains atomic mutations, the clock and the counter (all reset by + `data/reset` and `data/add`). + - A write answer module validates the body, applies the effect and renders + the response. + - Item reads of missing entities answer recorded 404s. + - The served catalog adds served write tools, with descriptions charged to + the discovery budget. +6. **Export.** Count served write tools in the bundle plan and README. +7. **Tests.** Domain, services, adapters and MCP-boundary coverage as the + spec's acceptance lists. +8. **Documentation.** Document the write model in the environment-state and + serving sections, and record the real-capture results, the AgentEnv run + and the live comparison in `verification.md`. + +There are no dependency, provider or REA tool contract changes beyond the +`environment-state` result members, and no new network activity. diff --git a/docs/specs/agent-environments/state-writes/spec.md b/docs/specs/agent-environments/state-writes/spec.md new file mode 100644 index 0000000000..e1bcba48ae --- /dev/null +++ b/docs/specs/agent-environments/state-writes/spec.md @@ -0,0 +1,427 @@ +# Apply write effects to the served state + +Issue: https://github.com/morluto/rea/issues/4933 + +## Outcome + +An environment served from its state also serves the write tools whose effects +the capture itself confirms: + +- a create adds an entity; +- an update changes the fields the call sends; +- a delete removes the entity; +- a document write changes a document. + +Later reads, list membership and `data/get` show the change, and `data/reset` +undoes it. + +The `environment-state` view decides which write tools are served. It replays +the capture's own timeline on the extracted initial state: each recorded write +applies its modelled effect, and each later read that the state answers is +rendered and compared with the recorded response. A write tool is served only +when at least one later read reproduces after its effect and none contradicts +it. Every other write tool stays unserved, listed with the reason. + +Values the application generates are deterministic: identities, sequences and +clock values follow the formats the capture recorded, so the same calls after +a reset give the same state. + +## Write plans + +The view adds two members to its result: + +- `writes`: one plan per manifest tool whose method may change state + (`changesState`); +- `write_clock`: the instant the served clock starts from. + +**Effects.** Each plan classifies its tool: + +| Effect | Tool | Target | +| ---------- | --------------------------------------------------------------------- | ------------------------- | +| `create` | `POST` to a state type's `collection_template` | an entity of that type | +| `update` | `PUT`, `PATCH` or `POST` to that template plus one trailing parameter | the entity the path names | +| `delete` | `DELETE` to that same item template | the entity the path names | +| `document` | `PUT`, `PATCH` or `POST` to a document view's path | that document | +| `none` | anything else | reason `no-state-target` | + +**Exemplar.** A plan is built from its first successful (2xx) recorded call, +the exemplar. + +- **Request body.** The exemplar's request body must be a JSON object that + could enter the state: a JSON-like media type, complete UTF-8 text within + 256 KiB, exact numbers, no `__proto__` member. Otherwise the reason is + `body-unusable`. A `DELETE` takes no body. +- **The written entity.** The entity is observed in the exemplar's response, + located by its identity value at the root, in one member, or in a + one-element array, as item views locate it. When the response does not carry + it, the entity is observed in the first later read that shows it: an item + read of that entity, or for a create, the one entity a collection read newly + lists. A create whose response names its id alone (a scalar, the identity + field, or one member) is observed only through a read of that id. With + neither, the reason is `no-observation`. + +**Field rules for creates.** Each field of the observed entity gets the first +rule that explains its value. Rules that read the call come first, so an +identity the client sends is an echo: + +1. **Echo.** A body member of the same name with an equal value. +2. **Copy.** A body string of 8 or more characters, found at a JSON pointer of + the body (the shallowest match). +3. **Path.** The value of a path parameter of the call, percent-decoded. + Serving decodes it from the collection the call names, whichever spelling + the call uses, and the gate fills a recorded call's path fields from that + collection too. +4. **Slug.** The slug of a body string member: lower case, runs of other + characters joined by `-`. +5. **Identity.** The type's `id_field` is generated. A sequence when it is an + integer (or a string of digits); otherwise a UUID when it has the + `8-4-4-4-12` hex shape; otherwise a token of the same length and character + classes. +6. **Clock.** A timestamp within 10 minutes of the exemplar's start time. The + format is kept: ISO-8601 with its separator, fraction digits and zone, or + epoch seconds or milliseconds. +7. **Sequence.** An integer one greater than the largest value of the field in + the collection, else in the type. +8. **Unique.** A UUID-shaped string that no other entity of the type shares. +9. **Template.** A string containing a generated value as a delimited token + (for example a URL that ends in the new id). It is rebuilt around the new + value. +10. **Constant.** Otherwise the exemplar's value. + +When a constant, or the text a template keeps, contains a path value of the +exemplar's call as a delimited token, the create is served only for calls +naming the exemplar's collection. + +**Body members of a create.** A member is either accepted or not: + +- **Accepted.** An echoed member, or the identity field's value. Every member + the exemplar sent is required. +- **Not accepted.** A member the application stored with a different value + (transformed), or one it did not show. A call that sends such a member is a + tool error naming the accepted members. + +**Updates.** Body members follow the same accepted and not-accepted rules, +except that a member the application stored differently is accepted only with +the value a recorded call sent it with, and then stores the value that call +recorded (its `constant` rule carries it). The type's identity field never +takes a rule: sent as the path's id it is the identity member, sent as another +value the application ignored it is accepted with that value and stores +nothing, and a stored id that changed is `unexplained-change`. A plan whose +calls use a member both as the identity and as a fixed member, or fix it at +different values, is `unexplained-change`, as serving could not take every +recorded call. So is an update a later read shows dropped a field the +entity held, since serving keeps every field an update does not write; a write's +own response may leave fields out. A member a later call is the first to store, +which the plan's observation does not show, keeps the JSON type that call stored +it as (the body's `kinds`), as an exemplar's member keeps the observation's. +`PUT` requires every member the exemplar sent; `PATCH` and `POST` require at +least one. A field the exemplar's update changed without sending it is +explained as one of: + +- clock; +- an increment by one; +- a slug of an echoed member. + +A field the entity lacked before is a change too when the view that observed +the update had shown the entity, so a field the application fills in on update +follows its rule. Any other change is `unexplained-change`. An increment or a slug applies to +every call that could take it, so the plan is also `unexplained-change` when +such a rule would give a call of the tool another value than the one it +recorded, or when a later call treats a member otherwise than the plan: one +call echoes it and another stores something else, or the two rule it +differently. + +**Deletes.** A delete removes the entity. Its response is the exemplar's +response, with the entity re-rendered when the response carries it. + +**Documents.** A document write merges the body's members (replaces them, for +`PUT`) into the document object that holds most of them: the root or one +member. Later calls write at the exemplar's object. Document reads answer only on their recorded line, so a write is +served only at the path the state holds the document at, and the gate treats a +recorded write at another path as an unmodelled write. A recorded write whose +body serving would refuse (a member of another JSON type than the document +holds) is `unexplained-change`. + +**Responses.** A write answers with the status and content type of one +recorded response: + +| Response | Body | +| ---------- | --------------------------------------------------------------------------------------------------------- | +| `entity` | the written entity, rendered into the recorded envelope at the plan's pointer through the recorded fields | +| `identity` | the recorded body with the new identity at the plan's pointer | +| `empty` | no body | +| `recorded` | the recorded body | + +A rendered body missing a field the recorded response carried is a tool error, +and the state is left unchanged. + +**List position.** A create's `insert` rule places the new entity in its +collection's order: sorted by the identity field or a clock field (ascending +or descending), `first` or `last`. The rule is the first candidate consistent +with both the initial `all`-membership recordings and the position a later +recorded list shows. With no evidence, the rule is `last`. + +## The gate + +The view replays every primary-origin record in timeline order on a copy of +the initial state. + +**Writes.** Only a 2xx write is modelled. A modelled write applies its effect, using the values it recorded +for generated fields and the stored values for fields it transformed. A +modelled write whose call the plan does not explain brings its entity to the +recorded outcome instead, without attribution. So does an update of an unknown +entity (below): its call cannot be checked, so it adds nothing to a built plan. +A later create whose generated +values take another form than serving gives them (a sequence value out of +order, a clock in another format, a non-UUID unique value) makes the plan +`unexplained-change`; integer identities are left to the id mapping. An update or +delete names its entity as an item read does (below). A later update that does +not answer with its entity is observed only by a read whose view shows every +clock the plan learned, since serving stamps them; without one it is unmodelled. + +Any other write that may have changed state, including a redirected or +unanswered one, is unmodelled, and what it may have changed +depends on where its path lies relative to each type's collection paths +(segment-prefix related, parameters matching any segment): + +- **Below an item path:** the named entity is unknown until a read shows it + again. When the path names no held entity, every entity of that collection + is unknown. +- **At a collection path:** that collection is not checked again. +- **Above collection paths:** everything under the path is not checked again. + +The path names its entity as an update's does and its collection as a create's +does, so a plain spelling reaches a collection held under an encoded one; a +path above collections covers each collection whose segments are its own +verbatim or once percent-decoded. + +An item read of an unknown entity shows it again: the copy takes its recorded +values. A failed read that is no absence shows nothing, and the entity stays +unknown. Writes that changed it since its last check are checked only on the +members they sent, less those the view treats as volatile, which show nothing +of a write. + +**Reads.** A read is checked when the served environment would answer it +from the state: + +- an item read of a state-backed item view, of the entity serving would name: + in a held collection the read's path matches, each segment verbatim or once + percent-decoded and verbatim spellings first, the id percent-decoded once, + then as written; +- a list or document read on one of its view's checked request lines, + chosen as replay chooses the line for the read's call: each path parameter + the read's value verbatim or once percent-decoded, preferring verbatim + spellings, then recorded order, and the query values decoded, in any order. + A read with no query on a whole list falls back, as serving does, on a + checked line that only pages it. A line whose query repeats a name with + different values is never chosen, since no call's arguments can name it; one + repeating a name with one value is, as a call naming it matches both. A list + is compared as served from the collection of that line. A read on a page + (`limit`, `size`, `per_page` and the like, or a `_start`–`_end` range; a size + of zero is no bound on a line whose recording listed items under it, while its + other sizes and range still bound the page) of a + whole list checks nothing once the collection holds more entities than the + page, which serving then answers from its recording. +- A read made while a modelled write to its entity or collection was still + running, starting before that write ended, checks nothing: it may show the + state before or after the write. Nor does such a read show an unknown entity + again. +- A read a write's effect was learned from checks nothing of that write, but + still contradicts the other writes it exercises. + +**Comparison.** The read's answer is the full response serving would give from +the copy: an item read's entity rendered into its item view's base recording, +or a list read's members as serving would list them (so a member the copy no +longer holds fails the check) rendered into its line's base recording. It is +compared with the recording, as the read gate compares, with volatile fields taking the +observed values. A recorded 400, 404 or 410, or a 2xx with a JSON `null` +body, for an entity the copy does not hold agrees with it. A successful read +whose usable body no longer holds the entity or the list fails, as does one of +an entity lacking a field every recording of its view carried, which serving +refuses to answer with. + +**Attribution.** + +- A check exercises the write tools that changed its entity or collection + since the previous check of it, less those whose effect later writes erased. + An update's effect is the fields it sent or changed, derived ones included. + A later update overwrites the fields it sets (an increment adds to its field + instead), a recorded outcome every field, and a create or delete the whole + entity. A document write's effect is the members it merged at its + object, or everything under the object it replaces, the whole document at the + root; an entity's fields match by name only, whatever their names look like. A write with nothing left on an entity no + longer counts there, nor in a collection where no entity it changed keeps any + of its effect. A mismatch contradicts them. A pass confirms those it shows + something of: a create's or delete's membership, or a changed field the + read's view shows and does not treat as volatile; a read that shows none of + an update's changes confirms nothing of it, and leaves it for a later check + of that entity or collection. +- A mismatch contradicts each exercised write that no unmodelled write came + after its latest call on any of the read's entities or collections where its + effect survives, whatever earlier write is still pending. For a failed list read the unmodelled write must + also follow the last check and pending writes of each member that differs, so an older change elsewhere + does not excuse a newer write the list shows undone. An item read checks the writes an unmodelled + write did follow only on the members they sent, and a list with such a write + is not checked again: the unmodelled write may explain the rest. +- A mismatch on an entity or collection that no write changed contradicts, + as `side-effect`, the modelled writes since that previous check whose paths + are segment-prefix related to the read's path. +- After a failed item check, the copy takes the read's recorded values (or + drops the entity on a recorded absence). A list whose check failed is not + checked again. +- A list that fails only for where one create placed its entity passes when + another of that create's insert candidates reproduces it, which then + becomes the plan's rule, unless an earlier list check relied on the rule + (an item read's confirmation does not fix it). +- The read that supplied a create's or update's observation does not check + it, but checks the other writes it exercises. + +**Verdicts.** A plan is served when at least one check confirms it and none +contradicts it. Otherwise the reason, in this precedence, is: + +1. the first build reason; +2. `contradicted`, with `contradicted_by` naming the first contradicting read; +3. `side-effect`; +4. `unconfirmed`; +5. `incomplete-entities`: an update or delete that answers with the entity, + when no initial entity holds every field its recorded answer shows (for + example a delete that answers with an edit view of the entity the reads + never showed), so it could answer only for entities the environment itself + created. An update's answer renders the entity it wrote, so a field one of + its rules writes counts as held, except an increment's, which adds only to + a held value. + +`checked_record_ordinals` lists the confirming reads. + +## Serving + +**Catalog.** In state mode, the catalog adds the write tools whose plan is +served. Their descriptions say the tool: + +- applies its effect to the served state; +- names its target type; +- lists its required and accepted body members, quoted and sampled within the + description budget as response fields are. + +The discovery budget charges them. `--replay-only` and the replay fallback are +unchanged. + +**Calls.** A served write answers synchronously against the live store and +changes it atomically, so every MCP session and `data/get` see one state. + +**Refusals.** These are tool errors that leave the state unchanged: + +- a body that is not an object; +- a missing required member; +- a member that is not accepted; +- a value of another JSON type than the exemplar's (for a member only a later + update stored, that call's), for documents as well as entities; +- an update or delete of an entity the state does not hold; +- a create in another collection when the plan is tied to its own; +- a document write at a path other than the one the state holds it at; +- a body member or a resulting value `data/add` would refuse (a number not + exact as `data/get` writes it, a `__proto__` member, nesting past the + limit), increments included, so `data/get` always loads back; +- a create, update or document write that would take what `data/get` reports + past the 32 MiB `data/add` intake limit, as a `resource_constraint` error; +- a rendered response missing a recorded field. + +When the tool, or else an item view of the type, recorded a 400, 404 or 410 +response, an update or delete of a missing entity answers with the latest such +recording instead. + +A create in a collection held under more than one spelling (`acme1` and +`%61cme1`) goes to the one the call spells verbatim, as item calls choose their +entity; a collection keeps its spelling after its last entity is deleted. A +reset or load knows the spellings of the Evidence's state, of the state it +installs and of the type's checked list lines (so a collection the capture +listed empty keeps its spelling), never those of a state it replaced. When two +spellings fit a call equally, the one holding entities wins. `data/add` refuses a document that `data/get` would write back past the +32 MiB intake limit once it adds the members a document may omit, such as +`types` and the state Evidence id. A derived state `data/get` would write +past that limit is not served: the environment serves replay and reports the +state `unavailable`. + +**Clock and generators.** The store's clock starts at `write_clock` and +advances one second before each served write; clock fields take its value in +their recorded format. + +- **Sequences:** one more than the largest current value in their scope, the + largest the state started with at its last reset or load, and the largest + value they generated, so a deleted entity's value is not reused, as database + sequences do not reuse values. +- **UUIDs and tokens:** derived from a SHA-256 of the state Evidence id, the + tool, the field and a write counter, skipping values the type already holds. + +`data/reset` and `data/add` restart the clock, the counter and the +sequences. + +**Missing entities.** An item read of an entity the state does not hold +answers with the tool's latest recorded 400, 404 or 410 response, or 2xx JSON +`null` response, the absences the gate accepts, when it has one, labelled +`state.view: "missing"`. Otherwise it stays a tool error. + +**Lists of a whole collection.** Written entities matter only if lists show +them, so the read gate's `membership: "all"` also covers a line that only pages +its list (`limit`, `offset`, `page`, `per_page`, `perpage`, `page_size`, +`pagesize`, `size`, `start`, `_start`, `_end`) when its recording listed the +whole collection. A list call with no query argument answers from such a line +of an `all` view of the collection its path arguments name, with +`replay.exact_match` false; a collection with no such line answers from its +recording. A call that asks for a page +still answers from its recording. + +**Provenance.** Write answers carry `state: {source: "state", view: "create" | +"update" | "delete", type, collection, id}`, or `{source: "state", view: +"document-write", path}`. + +**Data plane and export.** + +- `data/get` reports written entities without record citations; they load + back through `data/add` unchanged. +- `environment-export` counts served write tools in the bundle's tool count + and README. Its check stays a read tool. + +## Not modelled + +These stay unserved, or are labelled in the description: + +- side effects on counters, relations and copies of entities in other + collections; +- inputs the application transforms; +- members the capture never sent; +- reordering after an update; +- membership of filtered or paged lists, which answer their recorded members; +- write paths the manifest left literal; +- writes whose request body the capture did not retain; +- referential rules on delete: a delete removes the entity alone, so an + application that refuses to delete a referenced entity, or deletes its + dependents with it, is not modelled; +- authorization. + +## Acceptance + +- **Domain tests:** classification, every field rule, request-body admission, + observation from a later read, the insert rules, every verdict and reason, + attribution and resynchronisation, and taint by an unmodelled write. +- **Services tests:** + - store mutations and their atomicity; + - every refusal; + - generated values and their determinism across reset; + - `data/get` round trips of written entities; + - the recorded 404 for missing entities; + - the catalog and descriptions in each mode. +- **Boundary test:** a write tool changes the next read over MCP. + +**Real captures.** + +- The view runs over every showcase capture within its budget, and the + verification reports, per capture, the write tools served and the reasons + for the rest. +- One environment, chosen at random among those serving a write tool, is + exported, run in the AgentEnv framework, and graded from `data/get` after an + agent's create, update and delete. +- The served model is compared with that application running live on the same + calls. + +No test reaches an external network. diff --git a/docs/specs/agent-environments/state-writes/tasks.md b/docs/specs/agent-environments/state-writes/tasks.md new file mode 100644 index 0000000000..c3d238ab67 --- /dev/null +++ b/docs/specs/agent-environments/state-writes/tasks.md @@ -0,0 +1,22 @@ +# Tasks + +- [x] Record the write model, the gate and the serving contract. +- [x] Admit request bodies and add the plan types. +- [x] Explain created and updated entities with field rules. +- [x] Apply create, update, delete and document effects in one shared module. +- [x] Simulate the capture's timeline and decide each plan's verdict. +- [x] Leave entities below an unmodelled write unknown until a read shows them + again, and check writes such a write follows only on the members they + sent. +- [x] Serve gated write tools from the live store with deterministic values. +- [x] Answer recorded 400, 404 and 410 responses for missing entities. +- [x] Let lines that only page a whole collection back `membership: "all"`, + and answer argument-less list calls from them. +- [x] Count served write tools in exports and describe them in bundle READMEs. +- [x] Add domain, services, adapters and boundary coverage. +- [x] Document the write model and serving. +- [x] Run the view over every showcase capture and report per-capture verdicts. +- [x] Export one randomly chosen environment, run write tasks in the AgentEnv + framework and grade them from `data/get`. +- [x] Compare the served model with the live application on the same calls. +- [x] Run static, formatting, duplication, documentation and test gates. diff --git a/docs/specs/agent-environments/state-writes/verification.md b/docs/specs/agent-environments/state-writes/verification.md new file mode 100644 index 0000000000..dfcb16bc03 --- /dev/null +++ b/docs/specs/agent-environments/state-writes/verification.md @@ -0,0 +1,376 @@ +# Verification + +Verified base: cbb7b613 (this branch before this file). Executed on macOS with +Node v22.23.1 (the repository pins 24.18.0 in `.nvmrc`, which was not +installed); the AgentEnv runs and the live comparison ran on a disposable Linux +VM. Every check below ran as written after the last source change. + +## Gates + +- `npm run build:cached`, `npm run check:fast` (typecheck and lint), + `npm run format:check`, `npx knip`, `npm run scan:todos`, `npm run docs:check` + and `npm run verify:test-discovery`: passed. +- `npm run jscpd`: 3.43% duplicated lines over 3538 files, under the 3.5% gate. +- `npx vitest run --project domain --project services`: 7407 passed, 5 + pre-existing skips. +- `npm run test:prepare`, then `npm run test:focused --` the new write MCP + boundary, the serve MCP, CLI and process boundaries, the export CLI + boundary, the analysis-view CLI and MCP boundaries, the compact input schema + boundary and the contract description tests: 10 files, 72 passed. +- `npm run mcp-catalog:check`: passed. + +New tests: `environmentStateWriteValues` (94), `environmentStateWriteExplain` +(101), `environmentStateWriteApply` (52), `environmentStateWrites` (16, the gate +through the view), `environmentStateWriteAnswer` (14, serving) and +`environmentServeWritesMcp` (1, the built CLI over MCP). The replay card pinned +by the serve MCP boundary changed with the response summaries tool descriptions +carry; its digest now matches the combined branch. + +## Real captures + +The `environment-state` view ran over the 50 showcase captures (45 HAR +inspections and 5 `capture_browser_scenario` captures), each within its budget in +8–110 ms. It served 33 write tools in 16 captures (11 creates, 12 updates, 10 +deletes). Of the other 150 state-changing tools, 102 name no state type or +document (most write to literal single-entity paths the manifest left +untemplated, or to collections whose reads are not state-backed), and 48 are +modelled but not served: + +- 15 unconfirmed (no later read checked them); +- 13 no-observation; +- 8 contradicted, for example a deleted Airbyte source and EspoCRM task still + readable, Immich and BookStack item reads showing fields the create + response lacked, and Sonarr's tag list ordering; +- 7 unexplained-change, such as BookStack's description_html and Paperless's + derived text colour; +- 4 side-effect, for example a Sonarr tag also listed by `/tag/detail`; +- 1 incomplete-entities: WordPress answers a delete with the post in its edit + context, which the reads never showed for seeded posts. + +| Capture | Served (create / update / delete) | Not served | +| ---------------- | --------------------------------- | ----------------------------------------------------------------------------------------------- | +| airbyte | 2 / 2 / 0 | create contradicted, delete contradicted, delete unconfirmed | +| baserow | none | no-state-target ×3 | +| bookstack | 0 / 0 / 1 | create contradicted, create side-effect, update unexplained-change | +| directus | none | no-state-target ×2 | +| espocrm | none | create unconfirmed, delete contradicted, update unexplained-change | +| ghost | none | no-state-target ×2 | +| gitea | 1 / 1 / 0 | no-state-target ×2 | +| grafana | none | create unconfirmed, no-state-target ×5 | +| grist | none | create no-observation, no-state-target ×5 | +| grocy | none | create no-observation, delete no-observation, update no-observation, no-state-target ×2 | +| healthchecks | none | create no-observation, delete no-observation, update no-observation, no-state-target ×2 | +| homebox | none | document no-observation, no-state-target ×3 | +| immich | 0 / 0 / 1 | create contradicted, update contradicted | +| jellyfin | none | no-state-target ×2 | +| keycloak | none | no-state-target ×1 | +| komga | none | create unconfirmed ×2, no-state-target ×4 | +| linkding | 2 / 2 / 2 | create unconfirmed, delete unconfirmed, update unconfirmed, no-state-target ×2 | +| listmonk | 1 / 1 / 1 | — | +| lychee | none | no-state-target ×4 | +| mailpit | none | document no-observation, no-state-target ×5 | +| mattermost | 1 / 1 / 1 | no-state-target ×3 | +| mealie | none | create no-observation, no-state-target ×2 | +| memos | none | no-state-target ×3 | +| metabase | 0 / 0 / 1 | create side-effect, update unexplained-change | +| miniflux | 0 / 1 / 0 | no-state-target ×3 | +| n8n | none | create unconfirmed, no-state-target ×6 | +| navidrome | none | create no-observation, no-state-target ×2 | +| netbox | 1 / 1 / 1 | — | +| nocodb | none | create no-observation, no-state-target ×4 | +| nomad | none | no-state-target ×4 | +| paperless | 0 / 0 / 1 | create unconfirmed, update unexplained-change, no-state-target ×3 | +| photoprism | 1 / 1 / 0 | delete unconfirmed | +| pinry | none | create unconfirmed, delete unconfirmed, update unexplained-change | +| pocketbase | none | no-state-target ×4 | +| redpanda-console | none | document no-observation, no-state-target ×5 | +| sentry | 1 / 0 / 0 | update unexplained-change, no-state-target ×3 | +| shlink | none | create unconfirmed, no-state-target ×1 | +| sonarr | 0 / 1 / 0 | create contradicted, create side-effect, update contradicted | +| tandoor | none | no-state-target ×4 | +| umami | 1 / 1 / 1 | — | +| vault | none | no-state-target ×7 | +| vikunja | none | no-state-target ×4 | +| wordpress | none | create side-effect, delete incomplete-entities, document unconfirmed, update unexplained-change | + +An in-process smoke derived each capture's served state and called every served +write tool once with a varied body: all 33 applied (status equal to the +recording), and every created or updated entity read back from the state; +deleted entities answered the recorded 400 or 404 where one was recorded. Eight +older published manifests no longer match the tool names the current view +derives, so their state is unavailable and they serve replay, as designed; +none of them has a served write tool. + +## End to end on a published environment + +Five published environments serve a create, an update and a delete (Linkding, +Listmonk, Mattermost, NetBox, Umami); `random.SystemRandom().choice` picked +NetBox (NetBoxEnv-REA, NetBox 4.7.3). + +- `rea environment-export` (packed from this branch) wrote the bundle; its + smoke task passed in AgentEnv framework 0.9.1334 with 42 served tools (39 + reads and `post_dcim_sites`, `patch_dcim_sites_by_id`, + `delete_dcim_sites_by_id`). +- Four write tasks, each graded by an `env_outcome_verifier` that reads the + environment's state through `/agentenv` `data/get` (not the agent's reply), + with the replay-qa agent over the Hugging Face router, one run per task and + model: + +| Task | gpt-oss-120b | GLM-5.3-Flash | Tool calls (gpt-oss) | +| ----------------------------------------------- | ------------ | ------------- | ------------------------------------------------------------ | +| Create a site in the Copper Valley region | 1.0 | 1.0 | get_dcim_regions, post_dcim_sites | +| Update a seeded site's facility and description | 1.0 | 1.0 | get_dcim_sites, patch_dcim_sites_by_id | +| Delete one seeded site and only it | 1.0 | 1.0 | get_dcim_sites, delete_dcim_sites_by_id | +| Create a site, then set its facility | 1.0 | 1.0 | get_dcim_regions, post_dcim_sites, patch_dcim_sites_by_id ×2 | + +- The bundle's 12 retrieval tasks (one per seeded object) passed 12/12 for both + models, as they did on the published replay bundle. + +## The served model against the live application + +The same NetBox 4.7.3 image ran on the VM, seeded by the capture's own seed +script (the same ids), and one scenario ran against both the live API and the +environment served from the capture. Only `created` and `last_updated` were +normalised; every other member had to be equal: + +| Step | Live | Environment | Equal | +| ----------------------------------------- | ---- | ----------- | ---------------------------- | +| List sites | 200 | 200 | yes | +| Create Tide Gauge Annex (id 5) | 201 | 201 | yes | +| Read it | 200 | 200 | yes | +| List after the create (sorted by name) | 200 | 200 | yes | +| Patch its description and facility | 200 | 200 | yes | +| Read after the patch | 200 | 200 | yes | +| Create Granite Spur (id 6) | 201 | 201 | yes | +| Patch a seeded site | 200 | 200 | yes | +| Delete Tide Gauge Annex | 204 | 204 | yes | +| Read the deleted site | 404 | 404 | yes | +| Create Ledger Point (id 7, not reusing 5) | 201 | 201 | yes | +| Delete a seeded site with devices | 409 | 204 | no | +| Final list | 200 | 200 | no (follows from the delete) | + +11 of 13 steps were equal. The difference is NetBox's referential protection: +it refuses to delete a site that devices reference, while the environment's +delete removes the entity alone. The capture only deleted a scratch site with +no dependents, so the gate could not see the rule; it is listed under what is +not modelled. + +## Not covered + +- A run of the hosted `real-agentenv` lane in CI (the AgentEnv runs here used + the published framework release on a VM). +- Document writes on a real capture: the four modelled ones are unserved. +- Collections tied to a create's own collection on a real capture, and UUID + or token identities through an AgentEnv run (both are covered by tests). +- Agent token usage and cost. + +## After merging the base branches and main + +The branch then merged the serving branch at 5a61853a, the coverage branch at +b4b7ebc7 (their review fixes) and main at edbd291a, then main at 17f581ff +(the analysis-view tool description string, which main changes often). The gates above ran again +and passed: `npm run jscpd` 3.44% over 3579 files; domain and services 7442 +passed with 5 pre-existing skips; the focused boundary set 72 passed. The +write verdicts over the 50 captures were unchanged plan by plan, and the +in-process smoke again applied all 33 served write tools. The AgentEnv runs +and the live comparison above ran on cbb7b613, before these merges. +After the second main merge: `npm run check:fast`, `npm run format:check`, +`npm run docs:check` and `npm run mcp-catalog:check` passed; the contracts, +analysis-view and environment tests 857 passed with 3 pre-existing skips; the +analysis-view and environment MCP and CLI boundaries 53 passed. + +## After the review round + +The branch then merged the serving branch at c00fde41, the coverage branch at +f6c756aa and main at 1629266e, and took the review fixes (82556cd5, 138ea4b1, +6d21b62e). The coverage branch quotes field names in response summaries, so the +replay card pinned by the serve MCP boundary is now 12,818 bytes; removing +those quotes gives back the previous 12,618-byte digest exactly. The gates ran +again and passed: `npm run check:fast`, `npm run format:check`, `npx knip`, +`npm run scan:todos`, `npm run docs:check`, `npm run verify:test-discovery`, +`npm run verify:module-boundaries` and `npm run mcp-catalog:check`; +`npm run jscpd` 3.47% over 3601 files; domain and services 7463 passed with 5 +pre-existing skips; the focused boundary set 75 passed; conformance 279 +passed. The gate is stricter, but the write verdicts over the 50 captures were +unchanged plan by plan, and the in-process smoke again applied all 33 served +write tools. The AgentEnv runs and the live comparison above ran on cbb7b613, +before these merges and fixes. + +A second review round then made the gate and serving agree in five more places +(reads matched to checked lines as serving matches calls, increments and slugs +held for every call, insert rules still learned after an item read, the +document body check in the gate, and decoded parent path values). Each new test +fails on the previous head and passes here. The same gates passed again +(domain and services 7468 passed with 5 pre-existing skips, the focused +boundary set 75, conformance 279, `jscpd` 3.47%), the write verdicts over the 50 +captures were again unchanged plan by plan, and the smoke applied all 33 served +write tools. + +A third round kept a checked line's path spelling (replay prefers a verbatim +one, so two spellings of one parent stay separate lines) and compared a later +slug rule with the values earlier calls recorded rather than only their member +names; both new tests fail on the previous head. A fourth round chose a read's +line as replay does, so a read spelled plainly is checked against the encoded +line that answers it, and its list against that line's collection; its test +fails on the previous head. + +A review of the fidelity branch built on this one found four more places: a +document write at another path is now unmodelled in the gate, a create picks +the collection the call spells verbatim, a missing entity is answered with a +recorded 2xx `null` the gate accepted as an absence, and `data/add` refuses a +document whose `data/get` form would exceed the intake limit. Each new test +fails on the previous head. The gates passed again (domain and services 7475 +with 5 pre-existing skips, boundaries 75, conformance, `jscpd` 3.46%), the +verdicts over the 50 captures were unchanged plan by plan, and the smoke +applied all 33 served write tools. + +A sixth round, after merging the serving and coverage branches and main +335c042d, settled five more findings: a read whose usable body no longer holds +the entity or list fails its check, an entity lacking a field its view always +carried confirms nothing, a write a later write replaced is not confirmed by +reads of the replacement, sequences never reuse a value the state started +with, and a collection keeps its spelling after it empties. Each new test fails +on the previous head. Over the 50 captures one capture's verdicts change: +Linkding's bundle create and patch, confirmed only by a list read after the +same bundle was deleted, are now unconfirmed, so 31 write tools are served (10 +creates, 11 updates, 10 deletes) and the in-process smoke applies all 31. The +gates passed again (domain and services, boundaries, conformance and `jscpd`). + +A seventh round settled two more findings. An update keeps its confirmation +when a later write overwrites only part of what it changed: a derived change, +such as an incremented version, counts as part of the update's effect, and a +later increment adds to the field rather than replacing it. A reset or load +forgets the collection spellings of the state it replaced. Both new tests fail +on the previous head. The verdicts over the 50 captures are unchanged plan by +plan, the in-process smoke applies all 31 served write tools, and the gates +passed again. + +An eighth round settled three more. A later write erases the fields of an +earlier effect it overwrites, so two writes that together overwrite all of an +update's changes leave it nothing to be confirmed by. A recorded outcome erases +what earlier writes changed on its entity. A load keeps the Evidence's own +collection spellings, so a create in a collection the loaded state does not +hold keeps its recorded spelling. Each new test fails on the previous head. The +verdicts over the 50 captures are unchanged plan by plan, the smoke applies all +31 served write tools, and the gates passed again. + +A ninth round made a create prefer a collection the state holds over an equally +fitting spelling known only from the Evidence, so a create after a load lands in +the loaded collection and reads back. Its test fails on the previous head; the +verdicts and the smoke are unchanged and the gates passed again. + +A tenth round settled six findings Greptile raised on the list-lines branch +against this branch's code: + +- a read with no query on a whole list is checked on the paging line serving + answers it from, and a line whose query repeats a name is never checked; +- a read made while a write to its targets was still running checks nothing; +- a later call that stores a member otherwise than the plan refuses the plan; +- a create keeps the spelling of a collection the capture listed empty; +- a wide array no longer overflows the stack while checking path-dependent + values; +- the guide counts served write tools among the catalog's. + +Each new test fails on the previous head. The verdicts over the 50 captures +are unchanged plan by plan, the smoke applies all 31 served write tools, and +the gates passed again. + +An eleventh round settled six more findings, five raised on this branch and one on the fidelity branch: + +- a passing read confirms only the writes it shows something of; +- an item read names its entity as serving does, percent-decoded spellings included; +- a read a write's effect was learned from still contradicts the other writes; +- a line repeating a query name with one value stays checkable; +- document writes join the effect tracking, so a replacement overwrites an earlier merge; +- a call naming a page that the collection has outgrown answers from its recording (`page-exceeded`), and the gate does not check such reads. + +Each new test fails on the previous head. The verdicts over the 50 captures are unchanged plan by plan, the smoke +applies all 31 served write tools, and the gates passed again. + +A twelfth round settled four findings Greptile raised on the list-lines branch against this branch's code: + +- a create that answers with its id alone is observed only through a read of that id; +- a later call that is the first to send the path's id in its body adds the identity member to the plan; +- the copy search walks a request body with an index, so a wide body no longer takes quadratic time (500,000 zeros: 63 s + before, 74 ms after); +- the sent values of fixed members a plan keeps are counted against the view's allowance. + +Each new test fails on the previous head. The verdicts over the 50 captures are unchanged plan by plan, the smoke +applies all 31 served write tools, and the gates passed again. + +A thirteenth round settled seven findings Greptile raised on this branch, the fidelity branch and the list-lines branch: + +- a passing read that shows nothing of a write leaves it for a later check, instead of clearing it; +- a read begun while a write to an unknown entity still ran neither checks nor shows the entity again; +- a later create whose sequence fields do not follow the learned sequence refuses the plan; +- a document replacement below the root overwrites only what it replaced; +- a built document plan writes later calls at its own object, even one sharing no member with the document; +- an update or delete finds its entity as an item read does, percent-decoded spellings included; +- a `_start`–`_end` range bounds a page as `limit` does, in serving and in the gate. + +Each new test fails on the previous head. The verdicts over the 50 captures are unchanged plan by plan, the smoke +applies all 31 served write tools, and the gates passed again. + +A fourteenth round settled two findings Greptile raised on the list-queries branch against this branch's code: + +- the type's identity field never becomes a constant: an update whose id member the application ignored no longer + writes the exemplar's id into the entity another call names; +- a page size of zero bounds nothing, so a whole line recorded with `limit=0` answers from the state. + +Both tests fail on the previous head. The verdicts over the 50 captures are unchanged plan by plan, the smoke applies +all 31 served write tools, and the gates passed again. + +A fifteenth round settled nine findings Greptile raised on this branch, the fidelity branch and the list-lines branch, +five of them on the previous rounds' own changes: + +- an entity field whose name looks like a pointer is matched by name; only a document's pointers cover the members + under them; +- a failed read contradicts each exercised write no unmodelled write came after, rather than skipping all of them + when the oldest pending write came before one; +- an item read, update or delete names its entity as serving does, decoding the id once, so `book%2531` no longer + reaches `book1`; +- a plan whose calls use a member both as the identity and as a fixed member, or fix it at different values, is + refused; +- a size of zero bounds nothing only on a line whose recording listed items under it; +- an update whose observation drops a field the entity held is refused; +- a later create whose clock takes another format, or whose unique value is not a UUID, is refused; +- a derived state `data/get` would write past 32 MiB is not served. + +Each new test fails on the previous head. The verdicts over the 50 captures are unchanged plan by plan, the smoke +applies all 31 served write tools, and the gates passed again. + +A sixteenth round settled four findings Greptile raised on the fifteenth round's changes, here and on the fidelity, +list-lines and list-queries branches: + +- a size of zero is no bound only by itself: a range or another size on the same line still bounds the page; +- an update is refused for a dropped field only when a later read shows it dropped, since a write's own response may + leave fields out; +- interference is weighed against a tool's latest call on the read's entity or collection, not its latest call + anywhere. + +Each new test fails on the previous head. The verdicts over the 50 captures are unchanged plan by plan, the smoke +applies all 31 served write tools, and the gates passed again. + +A seventeenth round settled two more findings Greptile raised on the list-queries branch against this branch's +interference rule: an unmodelled write counts against a tool when it followed the tool's call on any target the read +covers, so a later call on another entity does not hide it; and a failed item read contradicts the writes no +unmodelled write followed, checking only the others on their members. Both tests fail on the previous head. The +verdicts over the 50 captures are unchanged plan by plan, the smoke applies all 31 served write tools, and the gates +passed again. A follow-up weighs interference only on targets where the tool's effect survives, so a call +whose effect a later write overwrote no longer excuses a later mismatch; its test fails on the previous head. + +## Collection-history accounting check + +On 2026-10-22, Node 24.18.0, I compared the source before the write-history +change (`cfb42ddde`) with the bounded-history implementation. Each operation +inserted an entity at a new collection path with a 128-character repeated +suffix and immediately deleted it. Across five fresh Vitest worker processes, +the median for 1,000 completed cycles was 14.57 ms wall / 23,867 us CPU before +and 9.98 ms / 16,914 us after. The process RSS samples were 381.1–387.9 MiB +before and 380.9–387.6 MiB after. This fixture benchmark measures one matched +completed trace, not a representative user task or process-family peak memory. +The values include Vitest and V8 and are noisy; they do not establish a memory +bound. The 1 MiB policy is a deterministic accounting allowance for +additional identifier history, set to 1/32 of the separate 32 MiB state-intake +ceiling; it is not derived from RSS samples and does not bound total store or +process memory. diff --git a/docs/testing.md b/docs/testing.md index 2517d62c91..c280643b6e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1818,7 +1818,7 @@ the built CLI, packs the checkout with `npm pack`, exports the bundle with registry package, and runs `agent-env run` on the bundle. It checks that the export reports the planned file set with the inputs copied byte-exact and a bundle served from the derived state, which serves fewer tools than the -manifest because write tools are not served, that the runner exits `0` with +manifest because write tools the capture did not confirm are not served, that the runner exits `0` with `tasks/smoke.json` reported as passed and the verifier's description naming that served tool count, and that no container exists after the run that did not exist before; the container inventory runs even when diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index 107e2b748b..b25ab0e7ee 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -595,12 +595,12 @@ Every entity cites its `record_ordinals`. `views` lists one plan per read tool in tool order: -| Shape | Plan | -| ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `item` | The entity object at `pointer` (the root, one member, or `/0` or `//0`), the ordered `fields` the view shows and the `required_fields` every recording had, and `envelope_ordinal`: the recording whose response, with the entity replaced, every item response shares | -| `list` | The array at `pointer`, its `fields`, `membership` (`all`: every entity of the collection in state order, when each recording listed exactly that, had no query, no item-total member (`total`, `count`, `totalCount`, `item_count`, `totalcount`, `itemcount`, but not `total_pages` or `per_page`; a number or a string of digits) differing from the number listed, and no `has_more`, `next` or next-page token member set; `recorded`: the entities each recording listed) and `counters` that equal the number listed | -| `document` | The latest initial value of a list tool whose responses hold no identified list, when its initial reads share one request line; held in `documents` with that `path`, whether or not the capture writes it, unless its value changed across any recorded write between those reads (`fields-differ`) | -| `recorded` | A `reason` the tool replays its recordings instead | +| Shape | Plan | +| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `item` | The entity object at `pointer` (the root, one member, or `/0` or `//0`), the ordered `fields` the view shows and the `required_fields` every recording had, and `envelope_ordinal`: the recording whose response, with the entity replaced, every item response shares | +| `list` | The array at `pointer`, its `fields`, `membership` (`all`: every entity of the collection in state order, when each recording listed exactly that, had no query or only paging parameters (`limit`, `offset`, `page`, `per_page`, `perpage`, `page_size`, `pagesize`, `size`, `start`, `_start`, `_end`), no item-total member (`total`, `count`, `totalCount`, `item_count`, `totalcount`, `itemcount`, but not `total_pages` or `per_page`; a number or a string of digits) differing from the number listed, and no `has_more`, `next` or next-page token member set; `recorded`: the entities each recording listed) and `counters` that equal the number listed | +| `document` | The latest initial value of a list tool whose responses hold no identified list, when its initial reads share one request line; held in `documents` with that `path`, whether or not the capture writes it, unless its value changed across any recorded write between those reads (`fields-differ`) | +| `recorded` | A `reason` the tool replays its recordings instead | An item or list plan is kept only when it reproduces the latest initial recording of each request line from the state, comparing JSON values without @@ -618,20 +618,65 @@ when its content type is JSON-like, it is UTF-8 text of at most 256 KiB, every number survives native conversion exactly, and no member is named `__proto__`. +`writes` lists one plan per tool whose request may change state, in tool +order. Its `effect` is `create` for a `POST` to a type's +`collection_template`, `update` for a `PUT`, `PATCH` or `POST` to that template +plus one trailing parameter, `delete` for a `DELETE` to it, `document` for a +`PUT`, `PATCH` or `POST` to a document's path, and otherwise `none` with reason +`no-state-target`. A plan is built from the tool's first successful call: its +request body must be a usable JSON object (as response bodies are usable), and +the written entity must show in that call's response or in a later read before +the next write. Each field of a created entity gets the first rule that +explains it: generated `identity` (a sequence, UUID or token, by the recorded +value's shape), `echo` of a body member, `copy` of a body string of 8 or more +characters, a `path` value, a `slug` of a body member, a `clock` value within 10 +minutes of the request in its recorded format, a `sequence` one past the +largest value of its collection or type, a `unique` UUID, or a `template` around +a generated value; other fields keep the observed value. An update's members +echo when stored as sent; a member stored differently is accepted only with its +recorded value (`body.fixed`) and stores the value the call that showed it +recorded (the `value` of its `constant` rule), and a field it changed without +sending must be a clock, an increment or a slug, or the plan is +`unexplained-change`; an increment or slug must hold for every call of the +tool. Every member the first call sent is required, except by `PATCH` and +`POST` updates. + +The view then replays the timeline on a copy of the initial state. Modelled +writes apply their effect; only a `2xx` write is modelled or an example, and +any other write (a redirect, an unanswered or unmodelled one) leaves the entity +below whose item path it wrote unknown until a read shows it again (a whole +collection when the path names no held entity), and taints what it wrote at or +above. A read the served environment would answer from the state is compared +with its recording as the read plans are, as the whole response it would serve: +the entity or list rendered into the envelope of the item view's or the line's +base recording, with the members the state still holds. A 400, 404 or 410 (or a +`null` body) agrees with an entity the copy does not hold. A match confirms +the writes that changed its entity or collection since it was last read; a +mismatch contradicts them, or, when an unmodelled write came after them, is +checked only on the members they sent; a mismatch no write explains marks the related writes since as +`side-effect`. A plan is `served` when a read confirmed it and none +contradicted it; otherwise its `reason` is the first of its build reason +(`no-call`, `body-unusable`, `no-observation`, `unexplained-change`), +`contradicted` (with `contradicted_by`), `side-effect`, `unconfirmed` and `incomplete-entities` (an update or delete that answers with the entity, when no initial entity holds every field its recorded answer shows, counting for an update the fields its rules write, so it could answer only for entities the environment created). +`checked_record_ordinals` lists the confirming reads, `insert` places created +entities (sorted by a field the collection's recorded order shows, else last), +and `write_clock` is the end of the last timed record, to the next second. + Values in the state are recorded data, so the view refuses the same parents as the manifest, including those inspected with `sensitive_values`, and a `capture_browser_scenario` parent must have selected response bodies. Types and identity are inferences, so the Evidence is always inferred. `unknowns` states that the state establishes no authorization model, no way the application -computes responses or derived values, and no effects of writes. The state +computes responses or derived values, and write effects only as modelled from +recorded calls. The state accumulator charges the manifest's admission and grouping, every record line, each body as temporary bytes before it is decoded and parsed, and every value it keeps, within the same 96 MiB budget. Documents hold whole recorded response values whose recorded text and rendered JSON are each at most 256 KiB, so a -capture with many read-only paths yields a correspondingly larger state result. The view performs no network activity -and serves nothing; `rea environment-serve` derives it from the capture at -startup, serves read tools from it and loads replacement states through the -AgentEnv data plane, as +capture with many read-only paths yields a correspondingly larger state result. +The view performs no network activity and serves nothing; `rea environment-serve` +derives it from the capture at startup, serves read tools and the served write +plans from it and loads replacement states through the AgentEnv data plane, as [serve from the state](agent-environments.md#serve-from-the-state) describes. ## Resource and lifecycle boundaries diff --git a/src/application/environment/environmentBundle.test.ts b/src/application/environment/environmentBundle.test.ts index 276dcfcf02..070caef420 100644 --- a/src/application/environment/environmentBundle.test.ts +++ b/src/application/environment/environmentBundle.test.ts @@ -22,7 +22,7 @@ import { } from "./environmentBundle.js"; import type { EnvironmentSource } from "./environmentSource.js"; import { prepareServedState } from "./environmentState.js"; -import { environmentToolAnswer } from "./environmentStateAnswer.js"; +import { environmentToolAnswer } from "./environmentToolAnswer.js"; import { EnvironmentStateStore } from "./environmentStateStore.js"; const fixture = environmentServeFixture(); @@ -218,7 +218,9 @@ describe("planEnvironmentBundle", () => { expect(readme).not.toContain("agent-env run grafana-example"); expect(readme).toContain("## What the environment serves"); expect(readme).not.toContain("What replay means"); - expect(readme).toContain("write tools are not served yet"); + expect(readme).toContain( + "write tools are served only where the capture's later reads confirmed\ntheir effect", + ); expect(readme).toContain( "`rea environment-serve --replay-only` serves pure replay", ); diff --git a/src/application/environment/environmentBundle.ts b/src/application/environment/environmentBundle.ts index c5fce6f373..0e85d00919 100644 --- a/src/application/environment/environmentBundle.ts +++ b/src/application/environment/environmentBundle.ts @@ -7,7 +7,7 @@ import { servedCatalog, type PreparedState } from "./environmentState.js"; import { environmentToolAnswer, type EnvironmentToolAnswer, -} from "./environmentStateAnswer.js"; +} from "./environmentToolAnswer.js"; import { EnvironmentStateStore } from "./environmentStateStore.js"; import type { EnvironmentToolDefinition } from "./environmentTools.js"; @@ -52,7 +52,7 @@ export interface EnvironmentServedCheck { export interface EnvironmentBundlePlan { readonly name: string; readonly packageSource: EnvironmentPackageSource; - /** Tools the bundle serves: every manifest tool for replay, the read tools from a state. */ + /** Tools the bundle serves: every manifest tool for replay; from a state, the read tools and the served write tools. */ readonly toolCount: number; readonly check: EnvironmentServedCheck | null; /** Whether the bundle serves from the state, as `environment-serve` reports it. */ @@ -282,7 +282,8 @@ server derives the state the capture observed, the \`environment-state\` view of the capture with the manifest's template rules, and serves from it: read tools answer from the state where it backs them and from their recordings otherwise, \`/agentenv\` serves \`data/reset\`, \`data/add\` and \`data/get\` on -it, and write tools are not served yet. When the state cannot be derived, or +it, and write tools are served only where the capture's later reads confirmed +their effect. When the state cannot be derived, or its catalog exceeds the discovery budget, the server logs why and replays instead: each MCP tool call selects the retained record whose path and query values best match the arguments and returns that record's recorded status, @@ -332,7 +333,7 @@ export const planEnvironmentBundle = ( // The catalog and answers the bundle's server will have, so the verifier and README predict them. const catalog = servedCatalog( source, - state !== undefined, + state, name, ENVIRONMENT_EXPORT_OPERATION, ); diff --git a/src/application/environment/environmentState.test.ts b/src/application/environment/environmentState.test.ts index ad36076289..eae81e0577 100644 --- a/src/application/environment/environmentState.test.ts +++ b/src/application/environment/environmentState.test.ts @@ -24,12 +24,16 @@ import { } from "../analysisView/AnalysisViewService.js"; import { loadEnvironmentSource } from "./environmentSource.js"; import { + admitServedState, deriveEnvironmentState, loadEnvironmentState, prepareServedState, } from "./environmentState.js"; import { STATE_INTAKE_LIMIT_BYTES } from "./environmentStateDocument.js"; -import { EnvironmentStateStore } from "./environmentStateStore.js"; +import { + EnvironmentStateStore, + ENVIRONMENT_COLLECTION_HISTORY_LIMIT_BYTES, +} from "./environmentStateStore.js"; const fixture = environmentStateServeFixture(); const { source, state } = environmentStateServeInputs(fixture); @@ -150,6 +154,89 @@ describe("loadEnvironmentState", () => { }); }); +describe("served collection history", () => { + it("bounds distinct paths added after reset while retaining baseline paths", () => { + const original = state.entities[0]; + if (original === undefined) throw new Error("Expected a seeded entity"); + const run = () => { + const store = new EnvironmentStateStore(state); + for (let index = 0; index < 2000; index += 1) { + const collection = `/api/history/${String(index)}-${"x".repeat(900)}`; + const id = `history-${String(index)}`; + const before = JSON.stringify(store.snapshot()); + const result = store.insert({ ...original, collection, id }, 0); + if (!result.ok) { + expect(JSON.stringify(store.snapshot())).toBe(before); + expect(result.error).toMatchObject({ + resource: "capacity", + retryable: false, + remediationAction: expect.stringContaining("fewer distinct"), + }); + expect(result.error.reason).toContain("accounting budget"); + return result.error.reportedLimits; + } + const inserted = store.entity(original.type, collection, id); + if (inserted === undefined) throw new Error("Expected inserted entity"); + store.remove(inserted); + } + throw new Error("Expected a history budget refusal"); + }; + const refused = run(); + if (refused === null) + throw new Error("Expected reported history budget details"); + expect(refused).toEqual(run()); + expect(refused).toMatchObject({ + max_collection_history_bytes: ENVIRONMENT_COLLECTION_HISTORY_LIMIT_BYTES, + }); + expect(refused.collection_history_bytes).toBeGreaterThan( + ENVIRONMENT_COLLECTION_HISTORY_LIMIT_BYTES, + ); + }); + + it("does not charge a deleted loaded collection again at the history limit", () => { + const original = state.entities[0]; + if (original === undefined) throw new Error("Expected a seeded entity"); + const loadedState = { + ...state, + entities: Array.from({ length: 1200 }, (_, index) => ({ + ...original, + collection: `/api/loaded/${String(index)}-${"y".repeat(80)}`, + id: `loaded-${String(index)}`, + })), + }; + const store = new EnvironmentStateStore(loadedState); + for (const entity of store.ofType(original.type)) store.remove(entity); + expect(store.knownCollectionsOf(original.type)).toHaveLength(1200); + + const baseline = loadedState.entities[0]; + if (baseline === undefined) throw new Error("Expected a loaded entity"); + let refused = false; + for (let index = 0; index < 2000; index += 1) { + const collection = `/api/generated/${String(index)}-${"z".repeat(900)}`; + const entity = { + ...original, + collection, + id: `generated-${String(index)}`, + }; + const inserted = store.insert(entity, 0); + if (!inserted.ok) { + refused = true; + break; + } + const held = store.entity(entity.type, collection, entity.id); + if (held === undefined) throw new Error("Expected generated entity"); + store.remove(held); + } + expect(refused).toBe(true); + + const restored = store.insert(baseline, 0); + expect(restored.ok).toBe(true); + expect( + store.entity(baseline.type, baseline.collection, baseline.id), + ).toEqual(baseline); + }); +}); + describe("deriveEnvironmentState", () => { it("derives the state inspect-analysis-view projects from the capture with the manifest's rules, under the same Evidence id", () => { const derived = deriveEnvironmentState(source, fixture.capture); @@ -272,6 +359,28 @@ describe("prepareServedState", () => { }); }); +describe("admitServedState", () => { + it("refuses a derived state data/get would write past what data/add loads back", () => { + expect(admitServedState(state, "environment_serve").ok).toBe(true); + // A euro sign is one UTF-16 unit but three bytes as data/get writes it. + const notes = "\u20ac".repeat(12_000_000); + const [first] = state.entities; + const large = { + ...state, + entities: [ + ...state.entities, + { ...first!, id: "large", fields: { ...first!.fields, notes } }, + ], + }; + const admitted = admitServedState(large, "environment_serve"); + if (admitted.ok) throw new Error("Expected the state to be refused"); + expect(projectAnalysisError(admitted.error)).toMatchObject({ + code: "resource_constraint", + message: expect.stringContaining("larger than data/add loads"), + }); + }); +}); + describe("EnvironmentStateStore", () => { it("reports its state as rea-state/v1, accepts that report back and resets to the initial state", () => { const store = new EnvironmentStateStore(state); diff --git a/src/application/environment/environmentState.ts b/src/application/environment/environmentState.ts index c1fd5fc53d..bd91e0e8e7 100644 --- a/src/application/environment/environmentState.ts +++ b/src/application/environment/environmentState.ts @@ -6,10 +6,11 @@ import { type StateType, type StateView, } from "../../domain/analysisView/environmentStateRules.js"; +import type { StateWritePlan } from "../../domain/analysisView/environmentStateWriteRules.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; -import type { - AnalysisInputIssue, +import { AnalysisResourceConstraintError, + type AnalysisInputIssue, } from "../../domain/analysisErrorCore.js"; import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; import { err, ok, type Result } from "../../domain/result.js"; @@ -22,6 +23,9 @@ import { type EnvironmentManifestTool, type EnvironmentSource, } from "./environmentSource.js"; +import { STATE_INTAKE_LIMIT_BYTES } from "./environmentStateDocument.js"; +import { initialStateBytes } from "./environmentStateStore.js"; +import { describeWrite } from "./environmentStateWriteDescription.js"; import { environmentToolDefinitions, type EnvironmentDataPlane, @@ -41,11 +45,22 @@ export interface EnvironmentState { readonly views: ReadonlyMap; readonly entities: readonly StateEntity[]; readonly documents: readonly StateDocument[]; + /** Write plans keyed by tool name. */ + readonly writes: ReadonlyMap; + /** The instant the served clock starts from, or null when the capture has no times. */ + readonly writeClock: string | null; } -/** Serving from the state is read-only: tools whose requests may change state are not served. */ -const servedFromState = (tool: EnvironmentManifestTool): boolean => - !changesState(tool.method, tool.path_template); +/** The write plans a catalog serves from: none before a state is derived. */ +type ServedWrites = Pick; + +/** Serving from the state serves read tools and the write tools whose effect the capture confirmed. */ +const servedFromState = ( + tool: EnvironmentManifestTool, + state: ServedWrites, +): boolean => + !changesState(tool.method, tool.path_template) || + state.writes.get(tool.name)?.served === true; /** The tools an environment serves and the data plane its card declares. */ export interface ServedCatalog { @@ -53,27 +68,36 @@ export interface ServedCatalog { readonly dataPlane: EnvironmentDataPlane; } -/** The catalog an environment serves: every manifest tool for replay, the read tools and the data plane from a state. */ +/** The catalog an environment serves: every manifest tool for replay; from a state, the read tools, the confirmed write tools and the data plane. */ export const servedCatalog = ( source: EnvironmentSource, - fromState: boolean, + state: ServedWrites | undefined, name: string, operation: string = ENVIRONMENT_SERVE_OPERATION, ): Result => { - const dataPlane: EnvironmentDataPlane = fromState ? "state" : "none"; - const catalogSource: EnvironmentSource = fromState - ? { - ...source, - manifest: { - ...source.manifest, - tools: source.manifest.tools.filter(servedFromState), + const dataPlane: EnvironmentDataPlane = + state === undefined ? "none" : "state"; + const tools = environmentToolDefinitions( + state === undefined + ? source + : { + ...source, + manifest: { + ...source.manifest, + tools: source.manifest.tools.filter((tool) => + servedFromState(tool, state), + ), + }, }, - } - : source; - const tools = environmentToolDefinitions(catalogSource, name, { - operation, - dataPlane, - }); + name, + { + operation, + dataPlane, + ...(state === undefined + ? {} + : { describe: (tool) => describeWrite(state.writes.get(tool.name)) }), + }, + ); return tools.ok ? ok({ tools: tools.value, dataPlane }) : tools; }; @@ -162,6 +186,8 @@ export const loadEnvironmentState = ( views: new Map(view.views.map((entry) => [entry.tool, entry])), entities: view.entities, documents: view.documents, + writes: new Map(view.writes.map((plan) => [plan.tool, plan])), + writeClock: view.write_clock, }); }; @@ -214,6 +240,29 @@ export interface PreparedState extends ServedState { >; } +/** The derived state, unless data/get would write it past what data/add loads back, so its own report could not be restored. */ +export const admitServedState = ( + state: EnvironmentState, + operation: string, +): Result => { + const bytes = initialStateBytes(state); + return bytes > STATE_INTAKE_LIMIT_BYTES + ? err( + new AnalysisResourceConstraintError( + operation, + "capacity", + "The derived state is larger than data/add loads, so it cannot be served", + { max_state_bytes: STATE_INTAKE_LIMIT_BYTES, state_bytes: bytes }, + { + retryable: false, + remediationAction: + "Capture fewer or smaller responses, or serve the environment as replay only.", + }, + ), + ) + : ok(state); +}; + const unavailable = (error: AnalysisError): PreparedState => { const { code, message } = projectAnalysisError(error); return { @@ -232,19 +281,20 @@ export const prepareServedState = ( } = {}, ): PreparedState => { const operation = options.operation ?? ENVIRONMENT_SERVE_OPERATION; - // The state catalog depends only on the manifest and name, so checking it first spares deriving a state that cannot be served. - const catalog = servedCatalog( - source, - true, - options.name ?? source.manifest.name, - operation, - ); - if (!catalog.ok) return unavailable(catalog.error); + const name = options.name ?? source.manifest.name; + // The read catalog depends only on the manifest and name, so checking it first spares deriving a state that cannot be served. + const reads = servedCatalog(source, { writes: new Map() }, name, operation); + if (!reads.ok) return unavailable(reads.error); const derived = deriveEnvironmentState(source, captureValue, operation); - return derived.ok + if (!derived.ok) return unavailable(derived.error); + const admitted = admitServedState(derived.value, operation); + if (!admitted.ok) return unavailable(admitted.error); + // Served write tools join the catalog, so the whole catalog is admitted again. + const catalog = servedCatalog(source, derived.value, name, operation); + return catalog.ok ? { state: derived.value, report: { served: true, evidence_id: derived.value.evidenceId }, } - : unavailable(derived.error); + : unavailable(catalog.error); }; diff --git a/src/application/environment/environmentStateAnswer.test.ts b/src/application/environment/environmentStateAnswer.test.ts index 32af8c9b35..7bc1eefe0c 100644 --- a/src/application/environment/environmentStateAnswer.test.ts +++ b/src/application/environment/environmentStateAnswer.test.ts @@ -267,6 +267,27 @@ describe("list answers", () => { }); }); + it("answers an unfiltered call from a paging-only line of the parent its path arguments name", () => { + const books = servingFrom([ + ["GET", "/api/orgs/1/books?limit=50", [{ id: 1 }, { id: 2 }]], + ["GET", "/api/orgs/2/books?limit=50", [{ id: 3 }]], + ["GET", "/api/orgs/3/books?limit=50", [{ id: 4 }]], + ]); + expect(answer(books, "get_orgs_books_by_id", { id: "2" })).toMatchObject({ + body: [{ id: 3 }], + replay: { record_ordinal: 1, exact_match: false }, + state: { + source: "state", + collection: "/api/orgs/2/books", + membership: "all", + listed: 1, + }, + }); + expect(answer(books, "get_orgs_books_by_id", { id: "9" })).toMatchObject({ + state: { source: "recording", reason: "unrecorded-request" }, + }); + }); + it("refuses a list whose member lacks a field every recording carried, never answering a partial body", () => { const serving = servingOf(); load(serving, (entities) => diff --git a/src/application/environment/environmentStateAnswer.ts b/src/application/environment/environmentStateAnswer.ts index 59f274d1c4..184e50fc15 100644 --- a/src/application/environment/environmentStateAnswer.ts +++ b/src/application/environment/environmentStateAnswer.ts @@ -2,6 +2,7 @@ import { stateBodyJson } from "../../domain/analysisView/EnvironmentStateAccumul import { collectionPathOf, listItems, + missingRequiredFields, renderStateEntity, renderStateList, replaceAt, @@ -14,6 +15,11 @@ import { type StateListView, type StateView, } from "../../domain/analysisView/environmentStateRules.js"; +import { + pageBound, + pagesOnly, +} from "../../domain/analysisView/environmentStatePaging.js"; +import { isAbsence } from "../../domain/analysisView/environmentStateWriteChecks.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import { AnalysisInputError, @@ -45,7 +51,11 @@ import { environmentToolArguments } from "./environmentTools.js"; export type StateProvenance = | { readonly source: "recording"; - readonly reason: RecordedViewReason | "no-view" | "unrecorded-request"; + readonly reason: + | RecordedViewReason + | "no-view" + | "unrecorded-request" + | "page-exceeded"; } | { readonly source: "state"; @@ -64,8 +74,16 @@ export type StateProvenance = } | { readonly source: "state"; - readonly view: "document"; + readonly view: "document" | "document-write"; readonly path: string; + } + | { + readonly source: "state"; + /** A write applied to the state, or the recorded answer for an entity it does not hold. */ + readonly view: "create" | "update" | "delete" | "missing"; + readonly type: string; + readonly collection: string; + readonly id: string; }; /** A replay answer that also says whether the state or a recording produced it. */ @@ -79,7 +97,7 @@ export interface StateServing { readonly store: EnvironmentStateStore; } -const refused = ( +export const refused = ( tool: EnvironmentManifestTool, userMessage: string, ): AnalysisError => @@ -94,12 +112,9 @@ const shownEntity = ( entity: StateEntity, volatile: ReadonlySet, ): Result => { - const shown = renderStateEntity(entity, view, volatile); - const missing = view.required_fields.filter( - (name) => !Object.hasOwn(shown, name), - ); + const missing = missingRequiredFields(entity, view, volatile); return missing.length === 0 - ? ok(shown) + ? ok(renderStateEntity(entity, view, volatile)) : err( refused( tool, @@ -122,11 +137,12 @@ const fromRecording = ( }; /** A recorded body the state renders into, usable exactly when extraction could use it. */ -const recordedJson = (exchange: RecordedExchange): JsonValue | undefined => - stateBodyJson(exchange.contentType, exchange); +export const recordedJson = ( + exchange: RecordedExchange, +): JsonValue | undefined => stateBodyJson(exchange.contentType, exchange); /** The rendered body on the base record's status, content type and provenance; never cut. */ -const stateAnswer = ( +export const stateAnswer = ( serving: StateServing, base: { readonly exchange: RecordedExchange; readonly exact: boolean }, body: JsonValue, @@ -157,22 +173,22 @@ const stateAnswer = ( }; /** One collection segment an item call names: a template literal, or a supplied path value. */ -interface CallSegment { +export interface CallSegment { readonly text: string; readonly supplied: boolean; } /** An item call's collection segments and identity value, kept as supplied rather than joined and re-split. */ -interface ItemCall { +export interface ItemCall { readonly segments: readonly CallSegment[]; readonly id: string; } -/** What an item call names, or undefined when a path argument is missing. */ -const itemCall = ( +/** A call's path segments: template literals and supplied values, or undefined when a path argument is missing. */ +export const callSegments = ( tool: EnvironmentManifestTool, args: ReplayArguments, -): ItemCall | undefined => { +): CallSegment[] | undefined => { const names = environmentToolArguments(tool).path; const params = new Map(tool.params.map((param) => [param.index, param])); const segments: CallSegment[] = []; @@ -189,12 +205,23 @@ const itemCall = ( if (typeof value !== "string") return undefined; segments.push({ text: value, supplied: true }); } - const last = segments.pop(); - return last?.supplied === true ? { segments, id: last.text } : undefined; + return segments; +}; + +/** What an item call names, or undefined when a path argument is missing. */ +export const itemCall = ( + tool: EnvironmentManifestTool, + args: ReplayArguments, +): ItemCall | undefined => { + const segments = callSegments(tool, args); + const last = segments?.pop(); + return segments !== undefined && last?.supplied === true + ? { segments, id: last.text } + : undefined; }; /** How many supplied segments a held collection matches verbatim, when it matches the call under replay's path rule. */ -const collectionMatch = ( +export const collectionMatch = ( collection: string, segments: readonly CallSegment[], ): number | undefined => { @@ -211,7 +238,7 @@ const collectionMatch = ( }; /** The entity an item call names, preferring verbatim collection spellings as replay does; the id is tried percent-decoded, then as written. */ -const namedEntity = ( +export const namedEntity = ( store: EnvironmentStateStore, type: string, call: ItemCall, @@ -232,7 +259,60 @@ const namedEntity = ( return undefined; }; -/** An item call renders the named entity into the recorded envelope; it never invents an entity. */ +/** The answer for an entity the state does not hold: the tool's latest recorded 400, 404 or 410, else a tool error. */ +export const missingAnswer = ( + serving: StateServing, + tool: EnvironmentManifestTool, + missing: { + readonly type: string; + readonly collection: string; + readonly id: string; + }, +): Result => { + // A read also answers with a 2xx JSON null, the absence the write gate accepts; a write never does, as a 2xx would + // report success for an entity that is not there. + const nullAbsence = tool.method === "GET"; + const latestAbsence = (ordinals: readonly number[]) => + ordinals + .map((ordinal) => serving.source.exchanges.get(ordinal)) + .filter( + (exchange): exchange is RecordedExchange => + exchange !== undefined && + (isAbsence(exchange.status) || + (nullAbsence && + exchange.status !== null && + exchange.status >= 200 && + exchange.status < 300 && + recordedJson(exchange) === null)), + ) + .at(-1); + // The tool's own recorded absence first, else that of an item view of the type. + const itemTools = new Set( + [...serving.store.state.views.values()].flatMap((view) => + view.shape === "item" && view.type === missing.type ? [view.tool] : [], + ), + ); + const recorded = + latestAbsence(tool.record_ordinals) ?? + latestAbsence( + serving.source.manifest.tools + .filter((candidate) => itemTools.has(candidate.name)) + .flatMap((candidate) => candidate.record_ordinals), + ); + if (recorded === undefined) + return err( + refused( + tool, + `The state holds no ${missing.type} entity ${missing.id} in ${missing.collection}.`, + ), + ); + return ok({ + ...replayAnswer(serving.source, recorded, false), + state: { source: "state", view: "missing", ...missing }, + }); +}; + +/** An item call renders the named entity into the recorded envelope; it never invents an entity; one the state does not hold answers the tool's recorded absence. */ const answerItem = ( serving: StateServing, tool: EnvironmentManifestTool, @@ -245,12 +325,11 @@ const answerItem = ( return err(refused(tool, `Tool ${tool.name} needs every path argument.`)); const entity = namedEntity(store, view.type, call); if (entity === undefined) - return err( - refused( - tool, - `The state holds no ${view.type} entity ${call.id} in /${call.segments.map(({ text }) => text).join("/")}.`, - ), - ); + return missingAnswer(serving, tool, { + type: view.type, + collection: `/${call.segments.map(({ text }) => text).join("/")}`, + id: call.id, + }); const shown = shownEntity( tool, view, @@ -297,18 +376,46 @@ const checkedLine = ( return selectReplayExchange(tool, lines, args)?.exchange; }; +/** For a list of a whole collection, a call that supplies no query argument answers from a checked line of the collection its path arguments name that only pages it, chosen as replay chooses; a requested page answers from its recording. */ +const unfilteredLine = ( + source: EnvironmentSource, + tool: EnvironmentManifestTool, + checked: readonly number[], + args: ReplayArguments, +): RecordedExchange | undefined => { + const query = environmentToolArguments(tool).query; + const segments = callSegments(tool, args); + if ( + segments === undefined || + [...query.values()].some((argument) => args[argument] !== undefined) + ) + return undefined; + const lines = new Map(); + for (const ordinal of checked) { + const exchange = source.exchanges.get(ordinal); + if ( + exchange !== undefined && + pagesOnly(exchange.query ?? undefined) && + collectionMatch(collectionPathOf(exchange, false), segments) !== undefined + ) + lines.set(ordinal, exchange); + } + return selectReplayExchange(tool, lines, args)?.exchange; +}; + type StateDocumentView = Extract; -/** One tool call: the tool and the arguments it was called with. */ +/** One tool call: the tool, the arguments it was called with, and whether they name the answering line exactly. */ interface ToolCall { readonly tool: EnvironmentManifestTool; readonly args: ReplayArguments; + readonly exact: boolean; } /** A list call renders the members of the checked line it names from the state. */ const answerList = ( serving: StateServing, - { tool, args }: ToolCall, + { tool, args, exact }: ToolCall, view: StateListView, base: RecordedExchange, ): Result => { @@ -320,6 +427,16 @@ const answerList = ( return fromRecording(source, tool, args, "body-unusable"); const collection = collectionPathOf(base, false); const type = store.type(view.type); + // A requested page the collection has outgrown is answered as recorded, never with more items than it asked for. + const size = exact + ? pageBound(base.query ?? undefined, recorded.length) + : undefined; + if ( + view.membership === "all" && + size !== undefined && + store.inCollection(view.type, collection).length > size + ) + return fromRecording(source, tool, args, "page-exceeded"); const members: readonly StateEntity[] = view.membership === "all" ? store.inCollection(view.type, collection) @@ -340,7 +457,7 @@ const answerList = ( } return stateAnswer( serving, - { exchange: base, exact: true }, + { exchange: base, exact }, renderStateList(view, envelope, items), { source: "state", @@ -376,16 +493,27 @@ const answerCheckedLine = ( view: StateListView | StateDocumentView, args: ReplayArguments, ): Result => { - const base = checkedLine( + const named = checkedLine( serving.source, tool, view.checked_record_ordinals, args, ); + const base = + named ?? + (view.shape === "list" && view.membership === "all" + ? unfilteredLine(serving.source, tool, view.checked_record_ordinals, args) + : undefined); if (base === undefined) return fromRecording(serving.source, tool, args, "unrecorded-request"); + // An unfiltered call answers on a line it did not name, so its replay match is not exact. return view.shape === "list" - ? answerList(serving, { tool, args }, view, base) + ? answerList( + serving, + { tool, args, exact: named !== undefined }, + view, + base, + ) : answerDocument(serving, tool, base); }; @@ -408,18 +536,3 @@ export const answerFromState = ( return answerCheckedLine(serving, tool, view, args); } }; - -/** How a served environment answers one tool call. */ -export type EnvironmentToolAnswer = ( - tool: EnvironmentManifestTool, - args: ReplayArguments, -) => Result; - -/** The answer a served environment gives: replay without a state store, the live state with one. */ -export const environmentToolAnswer = ( - source: EnvironmentSource, - store: EnvironmentStateStore | undefined, -): EnvironmentToolAnswer => - store === undefined - ? (tool, args) => replayEnvironmentTool(source, tool, args) - : (tool, args) => answerFromState({ source, store }, tool, args); diff --git a/src/application/environment/environmentStateCollections.ts b/src/application/environment/environmentStateCollections.ts new file mode 100644 index 0000000000..6f535eaf27 --- /dev/null +++ b/src/application/environment/environmentStateCollections.ts @@ -0,0 +1,50 @@ +import { collectionPathOf } from "../../domain/analysisView/environmentStateRules.js"; +import { + callSegments, + collectionMatch, + type StateServing, +} from "./environmentStateAnswer.js"; +import type { ReplayArguments } from "./environmentReplay.js"; +import type { EnvironmentManifestTool } from "./environmentSource.js"; + +/** The collections a create may go to: those the store knows, held ones first, then those the type's checked list lines read, so an empty one keeps its spelling. */ +const creatableCollectionsOf = ( + serving: StateServing, + type: string, +): readonly string[] => { + const listed: string[] = []; + for (const view of serving.store.state.views.values()) + if (view.shape === "list" && view.type === type) + for (const ordinal of view.checked_record_ordinals) { + const exchange = serving.source.exchanges.get(ordinal); + if (exchange !== undefined) + listed.push(collectionPathOf(exchange, false)); + } + return [...new Set([...serving.store.knownCollectionsOf(type), ...listed])]; +}; + +/** The collection a create writes to: a known collection the call's path matches, else the path as supplied. */ +export const createdCollection = ( + serving: StateServing, + tool: EnvironmentManifestTool, + args: ReplayArguments, + type: string, +): string | undefined => { + const segments = callSegments(tool, args); + if (segments === undefined || segments.some(({ text }) => text.length === 0)) + return undefined; + // The spelling the call supplies verbatim wins, as item calls choose their entity, then the first known: held ones lead. + let held: string | undefined; + let verbatim = -1; + for (const collection of creatableCollectionsOf(serving, type)) { + const matched = collectionMatch(collection, segments); + if (matched !== undefined && matched > verbatim) { + held = collection; + verbatim = matched; + } + } + return ( + held ?? + `/${segments.map(({ text, supplied }) => (supplied ? encodeURIComponent(text) : text)).join("/")}` + ); +}; diff --git a/src/application/environment/environmentStateDocument.ts b/src/application/environment/environmentStateDocument.ts index a0c11e93a6..985b138fa5 100644 --- a/src/application/environment/environmentStateDocument.ts +++ b/src/application/environment/environmentStateDocument.ts @@ -7,6 +7,7 @@ import { same, scalarText, segmentsOf, + type JsonObject, type StateDocument, type StateEntity, type StateType, @@ -16,9 +17,11 @@ import { type AnalysisInputIssue, } from "../../domain/analysisErrorCore.js"; import { canonicalJson } from "../../domain/comparisonSemantics.js"; +import { parseExactJson } from "../../domain/exactJsonNumber.js"; import { projectInputIssues } from "../../domain/inputIssueProjection.js"; import { jsonValueSchema, + jsonValueValidationIssue, ownedJsonValueSchema, type JsonValue, } from "../../domain/jsonValue.js"; @@ -195,6 +198,18 @@ const prototypeMemberPath = (root: unknown): Path | undefined => { return undefined; }; +/** Whether `data/add` accepts a value as a `fields` member or a document's `value`: bounded nesting, no `__proto__` member, and numbers exact as `data/get` writes them. */ +export const isLoadableValue = (value: JsonValue): boolean => + jsonValueValidationIssue(value) === undefined && + prototypeMemberPath(value) === undefined && + parseExactJson(JSON.stringify(value)) !== undefined; + +/** The first member of an object `data/add` would refuse as `fields`, if any: a `__proto__` member, or one whose value does not load. */ +export const unloadableMember = (object: JsonObject): string | undefined => + Object.keys(object).find( + (member) => member === "__proto__" || !isLoadableValue(object[member]!), + ); + /** The first schema problem, located without naming the caller's data. */ const schemaProblem = ( issues: readonly z.core.$ZodIssue[], diff --git a/src/application/environment/environmentStateStore.ts b/src/application/environment/environmentStateStore.ts index c3b34ac28b..b2c7b1e482 100644 --- a/src/application/environment/environmentStateStore.ts +++ b/src/application/environment/environmentStateStore.ts @@ -2,7 +2,12 @@ import type { StateEntity, StateType, } from "../../domain/analysisView/environmentStateRules.js"; -import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; +import { nextSequence } from "../../domain/analysisView/environmentStateWriteValues.js"; +import { + AnalysisInputError, + AnalysisResourceConstraintError, +} from "../../domain/analysisErrorCore.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; import { jsonParts } from "../../domain/jsonSerialization.js"; import { err, ok, type Result } from "../../domain/result.js"; import { compositeKey } from "../../domain/unicodeCodePointOrder.js"; @@ -15,11 +20,38 @@ import { type StateContent, } from "./environmentStateDocument.js"; -/** The UTF-8 bytes a value's JSON takes, counted part by part so it is never one string. */ -const jsonBytes = (value: unknown): number => { - let bytes = 0; - for (const part of jsonParts(value)) bytes += Buffer.byteLength(part, "utf8"); - return bytes; +/** Maximum logical budget for collection spellings added by served creates since reset/load. */ +export const ENVIRONMENT_COLLECTION_HISTORY_LIMIT_BYTES = 1024 * 1024; + +const collectionHistoryCost = ( + state: EnvironmentState, + type: string, + collection: string, +): number => { + // Account key bytes without building temporary key strings or a Set. + let sequenceBytes = 0; + for (const plan of state.writes.values()) { + if (!plan.served || plan.target !== type) continue; + for (const rule of plan.rules) { + if (rule.rule === "sequence" && rule.scope === "collection") + sequenceBytes += + 64 + + Buffer.byteLength(plan.tool, "utf8") + + 1 + + Buffer.byteLength(rule.field, "utf8") + + 1 + + Buffer.byteLength(collection, "utf8"); + else if (rule.rule === "identity" && rule.generator === "sequence") + sequenceBytes += + 64 + + Buffer.byteLength(type, "utf8") + + 1 + + Buffer.byteLength(rule.field, "utf8") + + 1 + + Buffer.byteLength(collection, "utf8"); + } + } + return 64 + Buffer.byteLength(collection, "utf8") + sequenceBytes; }; /** What `data/get` reports: the current state as a `rea-state/v1` document. */ @@ -33,10 +65,45 @@ export interface StateSnapshot { }; } +/** The UTF-8 bytes a value's JSON takes, counted part by part so it is never one string. */ +const jsonBytes = (value: unknown): number => { + let bytes = 0; + for (const part of jsonParts(value)) bytes += Buffer.byteLength(part, "utf8"); + return bytes; +}; + +/** The bytes JSON writes for these array items and the commas between them. */ +const itemsBytes = (items: readonly unknown[]): number => + items.reduce((sum, item) => sum + jsonBytes(item), 0) + + Math.max(0, items.length - 1); + +/** What `data/get` reports for some content of a state. */ +const snapshotOf = ( + state: EnvironmentState, + entities: readonly StateEntity[], + documents: readonly HeldDocument[], +): StateSnapshot => ({ + rea_state: { + version: 1, + state_evidence_id: state.evidenceId, + types: state.types, + entities, + documents, + }, +}); + +/** What `data/get` writes for a state as the Evidence holds it, in UTF-8 bytes, summed item by item. */ +export const initialStateBytes = (state: EnvironmentState): number => + jsonBytes(snapshotOf(state, [], [])) + + itemsBytes(state.entities) + + itemsBytes(state.documents); + /** * The live state one served environment answers from, shared by every MCP * session and the data plane. Mutations are synchronous and swap the whole - * state, so every answer reads one consistent state. + * state, so every answer reads one consistent state. A mutation that would take + * what `data/get` reports past the size `data/add` loads is refused, so that + * report always loads back. */ export class EnvironmentStateStore { #entities: ReadonlyMap = new Map(); @@ -45,20 +112,44 @@ export class EnvironmentStateStore { #documents: ReadonlyMap = new Map(); #content: StateContent = { entities: [], documents: [] }; readonly #types: ReadonlyMap; + readonly #clockStart: number; + /** What `data/get` writes for the current state, in UTF-8 bytes; kept by each mutation rather than reserialized. */ + #bytes = 0; + /** The bytes of `data/get` with no entities or documents. */ + readonly #frameBytes: number; + readonly #initialBytes: number; + /** Served writes since the state was last reset or loaded: the clock's seconds and the generators' counter. */ + #writes = 0; + /** The largest value each sequence has generated since the last reset or load, so a deleted entity's value is not reused. */ + readonly #sequences = new Map(); + /** The entities the state held when it was last reset or loaded, whose values no sequence reuses either. */ + #started: readonly StateEntity[] = []; + /** Baseline and post-start spellings kept known after their entities are deleted. */ + readonly #knownCollections = new Map>(); + /** Logical UTF-8 key bytes plus a 64-byte allowance per collection and potential sequence-key record. */ + #collectionHistoryBytes = 0; + /** The collections of the Evidence's own state, which stay known after any load. */ + #evidenceCollections: ReadonlyMap = new Map(); constructor(readonly state: EnvironmentState) { this.#types = new Map( state.types.map((type) => [type.collection_template, type]), ); + this.#clockStart = + state.writeClock === null ? 0 : Date.parse(state.writeClock); + this.#frameBytes = jsonBytes(snapshotOf(state, [], [])); + this.#initialBytes = initialStateBytes(state); this.reset(); } - /** Restore the initial state the Evidence holds. */ + /** Restore the initial state the Evidence holds; the clock and counter start again. */ reset(): void { - this.#install(this.state); + this.#install(this.state, this.#initialBytes); + this.#evidenceCollections = this.#typeCollections; + this.#restart(); } - /** Validate a caller-owned document and copy its JSON values before retaining them. */ + /** Validate a caller-owned document and copy its JSON values before retaining them; the clock and counter start again. */ replace(document: unknown): Result { return this.#replace(document, false); } @@ -75,7 +166,8 @@ export class EnvironmentStateStore { const content = parseStateDocument(this.state, document, owned); if (!content.ok) return content; // data/get writes members a document may omit, so the state it reports must still be one data/add loads. - if (jsonBytes(this.#snapshotOf(content.value)) > STATE_INTAKE_LIMIT_BYTES) + const bytes = this.#contentBytes(content.value); + if (bytes > STATE_INTAKE_LIMIT_BYTES) return err( new AnalysisInputError( ENVIRONMENT_SERVE_OPERATION, @@ -92,10 +184,200 @@ export class EnvironmentStateStore { ], ), ); - this.#install(content.value); + this.#install(content.value, bytes); + this.#restart(); return ok(undefined); } + /** The write a served call would be: its number since the last reset or load, and its clock instant, one second past the previous applied write. */ + pendingWrite(): { readonly count: number; readonly clockMs: number } { + const count = this.#writes + 1; + return { count, clockMs: this.#clockStart + count * 1000 }; + } + + /** Count an applied write and keep the sequence values it took, so a refused call takes none. */ + commitWrite(sequences: ReadonlyMap): void { + this.#writes += 1; + for (const [key, value] of sequences) { + const taken = this.#sequences.get(key); + // No earlier value leaves this one in place, including when it is negative. + this.#sequences.set( + key, + taken === undefined ? value : Math.max(taken, value), + ); + } + } + + /** The next value of a sequence: the next held integer, else one past the later of that and the last value taken. */ + sequenceAfter(key: string, held: number | undefined): number { + return nextSequence(held, this.#sequences.get(key)); + } + + #restart(): void { + this.#writes = 0; + this.#sequences.clear(); + this.#started = this.#content.entities; + this.#knownCollections.clear(); + this.#collectionHistoryBytes = 0; + for (const [type, collections] of this.#typeCollections) + for (const collection of collections) + this.#rememberCollection(type, collection); + for (const [type, collections] of this.#evidenceCollections) + for (const collection of collections) + this.#rememberCollection(type, collection); + } + + #rememberCollection(type: string, collection: string): void { + const known = this.#knownCollections.get(type) ?? new Set(); + known.add(collection); + this.#knownCollections.set(type, known); + } + + /** The entities of a type the state held when it was last reset or loaded. */ + startedWith(type: string): readonly StateEntity[] { + return this.#started.filter((entity) => entity.type === type); + } + + /** Every held entity of a type, in state order. */ + ofType(type: string): readonly StateEntity[] { + return this.#content.entities.filter((entity) => entity.type === type); + } + + /** Add an entity at `index` of its collection's order, unless the state would outgrow what `data/add` loads. */ + insert( + entity: StateEntity, + index: number, + ): Result { + const entities = this.#content.entities; + const bytes = this.#grown( + jsonBytes(entity) + (entities.length > 0 ? 1 : 0), + ); + if (!bytes.ok) return bytes; + const known = this.#knownCollections.get(entity.type); + const newCollectionBytes = + known?.has(entity.collection) === true + ? 0 + : collectionHistoryCost(this.state, entity.type, entity.collection); + if (newCollectionBytes > 0) { + const historyBytes = this.#collectionHistoryBytes + newCollectionBytes; + if (historyBytes > ENVIRONMENT_COLLECTION_HISTORY_LIMIT_BYTES) + return err( + new AnalysisResourceConstraintError( + ENVIRONMENT_SERVE_OPERATION, + "capacity", + "Served collection history would exceed its deterministic accounting budget", + { + max_collection_history_bytes: + ENVIRONMENT_COLLECTION_HISTORY_LIMIT_BYTES, + collection_history_bytes: historyBytes, + }, + { + retryable: false, + remediationAction: + "Reset the served state or load a smaller state to clear generated collection history, then use fewer distinct collection paths.", + }, + ), + ); + } + const members = this.inCollection(entity.type, entity.collection); + const anchor = + index < members.length + ? entities.indexOf(members[index]!) + : members.length > 0 + ? entities.indexOf(members.at(-1)!) + 1 + : entities.length; + this.#install( + { + ...this.#content, + entities: [ + ...entities.slice(0, anchor), + entity, + ...entities.slice(anchor), + ], + }, + bytes.value, + ); + if (newCollectionBytes > 0) { + this.#rememberCollection(entity.type, entity.collection); + this.#collectionHistoryBytes += newCollectionBytes; + } + return ok(undefined); + } + + /** Replace a held entity's values in place, unless the state would outgrow what `data/add` loads. */ + update(entity: StateEntity): Result { + const key = compositeKey([entity.type, entity.collection, entity.id]); + const held = this.#entities.get(key); + if (held === undefined) return ok(undefined); + const bytes = this.#grown(jsonBytes(entity) - jsonBytes(held)); + if (!bytes.ok) return bytes; + this.#install( + { + ...this.#content, + entities: this.#content.entities.map((candidate) => + candidate === held ? entity : candidate, + ), + }, + bytes.value, + ); + return ok(undefined); + } + + remove(entity: StateEntity): void { + const held = this.#entities.get( + compositeKey([entity.type, entity.collection, entity.id]), + ); + if (held === undefined) return; + const entities = this.#content.entities.filter( + (candidate) => candidate !== held, + ); + this.#install( + { ...this.#content, entities }, + this.#bytes - jsonBytes(held) - (entities.length > 0 ? 1 : 0), + ); + } + + /** Set a held document's value, unless the state would outgrow what `data/add` loads; a value that is not the Evidence's cites no record. */ + setDocument( + tool: string, + value: JsonValue, + ): Result { + const held = this.#documents.get(tool); + if (held === undefined) return ok(undefined); + const written = { tool: held.tool, path: held.path, value }; + const bytes = this.#grown(jsonBytes(written) - jsonBytes(held)); + if (!bytes.ok) return bytes; + this.#install( + { + ...this.#content, + documents: this.#content.documents.map((document) => + document === held ? written : document, + ), + }, + bytes.value, + ); + return ok(undefined); + } + + /** The state's bytes after a change of `delta`, or a refusal when a change that grows it would pass the intake limit. */ + #grown(delta: number): Result { + const bytes = this.#bytes + delta; + if (delta <= 0 || bytes <= STATE_INTAKE_LIMIT_BYTES) return ok(bytes); + return err( + new AnalysisResourceConstraintError( + ENVIRONMENT_SERVE_OPERATION, + "capacity", + "The write would take the served state past the size data/add loads", + { max_state_bytes: STATE_INTAKE_LIMIT_BYTES, state_bytes: bytes }, + { + retryable: false, + remediationAction: + "Delete entities, or restore a smaller state with data/reset or data/add.", + }, + ), + ); + } + type(collectionTemplate: string): StateType | undefined { return this.#types.get(collectionTemplate); } @@ -118,28 +400,43 @@ export class EnvironmentStateStore { return this.#typeCollections.get(type) ?? []; } + /** Every collection spelling in the active, baseline, Evidence, or generated history, with active entities first. */ + knownCollectionsOf(type: string): readonly string[] { + const known = new Set([ + ...this.collectionsOf(type), + ...(this.#knownCollections.get(type) ?? []), + ]); + return [...known]; + } + document(tool: string): HeldDocument | undefined { return this.#documents.get(tool); } snapshot(): StateSnapshot { - return this.#snapshotOf(this.#content); + return snapshotOf( + this.state, + this.#content.entities, + this.#content.documents, + ); } - #snapshotOf(content: StateContent): StateSnapshot { - return { - rea_state: { - version: 1, - state_evidence_id: this.state.evidenceId, - types: this.state.types, - entities: content.entities, - documents: content.documents, - }, - }; + /** What `data/get` writes for the current state, in UTF-8 bytes. */ + byteLength(): number { + return this.#bytes; } - /** Build every index before assigning any, so a reader never sees half a state. */ - #install(content: StateContent): void { + /** What `data/get` writes for some content, summed item by item so the whole state is never one string. */ + #contentBytes(content: StateContent): number { + return ( + this.#frameBytes + + itemsBytes(content.entities) + + itemsBytes(content.documents) + ); + } + + /** Build current-state indexes without copying baseline collections into generated history. */ + #install(content: StateContent, bytes: number): void { const entities = new Map(); const collections = new Map(); const typeCollections = new Map(); @@ -167,5 +464,6 @@ export class EnvironmentStateStore { content.documents.map((document) => [document.tool, document]), ); this.#content = content; + this.#bytes = bytes; } } diff --git a/src/application/environment/environmentStateWriteAnswer.test.ts b/src/application/environment/environmentStateWriteAnswer.test.ts new file mode 100644 index 0000000000..1df8da53f0 --- /dev/null +++ b/src/application/environment/environmentStateWriteAnswer.test.ts @@ -0,0 +1,941 @@ +import { describe, expect, it } from "vitest"; + +import { + servedFrom, + type Served, +} from "../../../tests/fixtures/environmentStateServing.js"; +import { + BOOK_INIT_ROWS, + BOOK_NOT_FOUND, + BOOK_ROWS, + DELETE_NOT_FOUND, + FIRST_BOOK, + SECOND_BOOK, + THIRD_BOOK, + book, + bookClock, + renamedBook, + type Book, +} from "../../../tests/fixtures/environmentStateWrites.js"; +import type { TimedCaptureRow } from "../../../tests/fixtures/analysisView.js"; +import { seededUuid } from "../../domain/analysisView/environmentStateWriteValues.js"; +import type { ReplayArguments } from "./environmentReplay.js"; +import { servedCatalog } from "./environmentState.js"; +import { + environmentToolArguments, + environmentToolDefinitions, +} from "./environmentTools.js"; + +/** The book the first served create of "New Book" makes: the next id, one second past the served clock's start. */ +const NEW_BOOK = book(3, "New Book", bookClock(13)); +const CREATE_NEW_BOOK = { body: { name: "New Book" } }; + +const entityOf = (books: Served, id: string) => + books.store.snapshot().rea_state.entities.find((entity) => entity.id === id); + +/** Load what data/get reports back through data/add: the state must come back byte for byte. */ +const expectReloadsUnchanged = (books: Served): void => { + const before = books.snapshot(); + expect(books.store.replace(JSON.parse(before)).ok).toBe(true); + expect(books.snapshot()).toBe(before); +}; + +describe("served creates", () => { + it("refuses an exhausted generated-value collision search without changing state", () => { + const existing = { + id: "123e4567-e89b-42d3-a456-426614174000", + ref: "223e4567-e89b-42d3-a456-426614174000", + name: "Existing", + }; + const created = { + id: "323e4567-e89b-42d3-a456-426614174000", + ref: "423e4567-e89b-42d3-a456-426614174000", + name: "Created", + }; + const books = servedFrom([ + ["GET", "/api/items", [existing]], + ["GET", `/api/items/${existing.id}`, existing], + ["POST", "/api/items", created, 201, { name: "Created" }], + ["GET", `/api/items/${created.id}`, created], + ["GET", "/api/items", [existing, created]], + ]); + const plan = books.store.state.writes.get("post_items"); + if (plan?.served !== true) + throw new Error("Expected post_items to be served"); + const rule = plan.rules.find((candidate) => candidate.rule === "unique"); + expect(rule).toMatchObject({ field: "ref", rule: "unique" }); + if (rule?.rule !== "unique") throw new Error("Expected a unique ref rule"); + const candidates = Array.from({ length: 65 }, (_, attempt) => + seededUuid( + `${books.state.evidenceId}:${plan.tool}:${rule.field}:1:${String(attempt)}`, + created.ref, + ), + ); + const snapshot = books.store.snapshot(); + const collisions = candidates.map((ref, index) => ({ + type: "/api/items", + collection: "/api/items", + id: `collision-${String(index)}`, + fields: { id: `collision-${String(index)}`, ref, name: "Collision" }, + view_fields: [], + record_ordinals: [], + })); + const loaded = { + ...snapshot, + rea_state: { + ...snapshot.rea_state, + entities: [...snapshot.rea_state.entities, ...collisions], + }, + }; + const replaced = books.store.replace(loaded); + if (!replaced.ok) throw new Error(JSON.stringify(replaced.error)); + const before = books.snapshot(); + const pending = books.store.pendingWrite(); + + const refusal = books.refusal(plan.tool, { body: { name: "Created" } }); + + expect(JSON.stringify(refusal)).toContain( + "Could not generate a unique value for ref after 64 attempts", + ); + expect(books.snapshot()).toBe(before); + expect(books.store.pendingWrite()).toEqual(pending); + }); + it("creates a book with the next id, the served clock and a derived slug, which later reads show", () => { + const books = servedFrom(); + const created = books.answer("post_books", CREATE_NEW_BOOK); + expect(created).toMatchObject({ + status: 201, + content_type: "application/json", + state: { + source: "state", + view: "create", + type: "/api/books", + collection: "/api/books", + id: "3", + }, + }); + expect(created.body).toEqual(NEW_BOOK); + expect(books.answer("get_books_by_id", { id: "3" })).toMatchObject({ + status: 200, + body: NEW_BOOK, + state: { source: "state", view: "item", id: "3" }, + }); + expect(books.answer("get_books")).toMatchObject({ + body: [FIRST_BOOK, SECOND_BOOK, NEW_BOOK], + state: { source: "state", view: "list", membership: "all", listed: 3 }, + }); + }); + + it("reports a created book in data/get without record citations and loads it back unchanged", () => { + const books = servedFrom(); + books.answer("post_books", CREATE_NEW_BOOK); + expect(entityOf(books, "3")).toEqual({ + type: "/api/books", + collection: "/api/books", + id: "3", + fields: NEW_BOOK, + view_fields: [], + record_ordinals: [], + }); + expectReloadsUnchanged(books); + expect(books.answer("get_books_by_id", { id: "3" }).body).toEqual(NEW_BOOK); + }); +}); + +describe("served updates and deletes", () => { + it("keeps an entity's identity when the application ignored the id an update sent", () => { + const renamed = renamedBook(FIRST_BOOK, "Renamed Book", 3); + const books = servedFrom([ + ...BOOK_INIT_ROWS, + [ + "PATCH", + "/api/books/1", + renamed, + 200, + { id: 999, name: "Renamed Book" }, + ], + ["GET", "/api/books/1", renamed], + ]); + const answer = books.answer("patch_books_by_id", { + id: "2", + body: { id: 999, name: "Second Edition" }, + }); + expect(answer.body).toMatchObject({ id: 2, name: "Second Edition" }); + expect(books.answer("get_books_by_id", { id: "2" }).body).toMatchObject({ + id: 2, + }); + }); + + const SECOND_EDITION = renamedBook(SECOND_BOOK, "Second Edition", 13); + const RENAME_SECOND = { id: "2", body: { name: "Second Edition" } }; + + it("changes only the sent members, the slug that follows them and the clock", () => { + const books = servedFrom(); + expect(books.answer("patch_books_by_id", RENAME_SECOND)).toMatchObject({ + status: 200, + body: SECOND_EDITION, + state: { source: "state", view: "update", id: "2" }, + }); + expect(books.answer("get_books_by_id", { id: "2" }).body).toEqual( + SECOND_EDITION, + ); + expect(books.answer("get_books").body).toEqual([ + FIRST_BOOK, + SECOND_EDITION, + ]); + }); + + it("cites no records for an updated book in data/get, so it loads back unchanged", () => { + const books = servedFrom(); + books.answer("patch_books_by_id", RENAME_SECOND); + expect(entityOf(books, "2")).toMatchObject({ + fields: SECOND_EDITION, + record_ordinals: [], + }); + expectReloadsUnchanged(books); + }); + + it("removes a deleted book; a later read answers the recorded 404 as a missing entity", () => { + const books = servedFrom(); + expect(books.answer("delete_books_by_id", { id: "2" })).toMatchObject({ + status: 204, + body: null, + state: { source: "state", view: "delete", id: "2" }, + }); + expect(books.answer("get_books_by_id", { id: "2" })).toMatchObject({ + status: 404, + body: BOOK_NOT_FOUND, + state: { + source: "state", + view: "missing", + type: "/api/books", + collection: "/api/books", + id: "2", + }, + }); + expect(books.answer("get_books").body).toEqual([FIRST_BOOK]); + }); + + it("answers an update or delete of a missing book with a recorded 404, else refuses it", () => { + const books = servedFrom(); + const before = books.snapshot(); + // The update recorded no absence, so it answers the item view's; the delete answers its own. + expect( + books.answer("patch_books_by_id", { id: "99", body: { name: "Ghost" } }), + ).toMatchObject({ + status: 404, + body: BOOK_NOT_FOUND, + state: { view: "missing", id: "99" }, + }); + expect(books.answer("delete_books_by_id", { id: "99" })).toMatchObject({ + status: 404, + body: DELETE_NOT_FOUND, + state: { view: "missing", id: "99" }, + }); + expect(books.snapshot()).toBe(before); + const unrecorded = servedFrom( + BOOK_ROWS.filter(([, , , status]) => status !== 404), + ); + for (const [name, args] of [ + ["patch_books_by_id", { id: "99", body: { name: "Ghost" } }], + ["delete_books_by_id", { id: "99" }], + ] as const) + expect(unrecorded.refusal(name, args)).toMatchObject({ + code: "invalid_request", + message: expect.stringContaining( + "holds no /api/books entity 99 in /api/books", + ), + }); + }); +}); + +describe("refusals", () => { + it("refuses a body the capture does not support and leaves the state unchanged", () => { + const books = servedFrom(); + const before = books.snapshot(); + const refused: [string, ReplayArguments, string][] = [ + [ + "post_books", + { body: { name: "Extra", color: "red" } }, + "Member color is not one the capture showed stored", + ], + ["post_books", { body: {} }, 'lacks required members "name"'], + [ + "post_books", + { body: { name: 42 } }, + "Member name must be a JSON string", + ], + ["post_books", { body: ["New Book"] }, "takes a JSON object body"], + [ + "patch_books_by_id", + { id: "1", body: "New Book" }, + "takes a JSON object body", + ], + ["patch_books_by_id", { id: "1", body: {} }, "sends none of the members"], + [ + "patch_books_by_id", + { id: "1", body: { title: "New" } }, + "Member title is not one", + ], + ]; + for (const [name, args, message] of refused) { + expect(books.refusal(name, args)).toMatchObject({ + code: "invalid_request", + message: expect.stringContaining(message), + }); + expect(books.snapshot()).toBe(before); + } + }); + + it("accepts a member the application stored differently only with the value the capture sent", () => { + const bound = (entry: Book, format: string): Book => ({ ...entry, format }); + const books = servedFrom([ + [ + "GET", + "/api/books", + [bound(FIRST_BOOK, "PAPERBACK"), bound(SECOND_BOOK, "PAPERBACK")], + ], + ["GET", "/api/books/1", bound(FIRST_BOOK, "PAPERBACK")], + ["GET", "/api/books/2", bound(SECOND_BOOK, "PAPERBACK")], + [ + "POST", + "/api/books", + bound(THIRD_BOOK, "HARDCOVER"), + 201, + { name: "Third Book", format: "hardcover" }, + ], + ["GET", "/api/books/3", bound(THIRD_BOOK, "HARDCOVER")], + ]); + const before = books.snapshot(); + expect( + books.refusal("post_books", { + body: { name: "Soft", format: "softcover" }, + }), + ).toMatchObject({ + message: expect.stringContaining( + "Member format is accepted only with the value the capture sent", + ), + }); + expect(books.snapshot()).toBe(before); + expect( + books.answer("post_books", { + body: { name: "Hard", format: "hardcover" }, + }).body, + ).toMatchObject({ id: 3, name: "Hard", format: "HARDCOVER" }); + }); + + it("holds a member a later update first stored to the JSON type it stored", () => { + const shelved = { ...SECOND_BOOK, tags: ["a"] }; + const renamed = renamedBook(FIRST_BOOK, "Renamed Book", 3); + const retagged = { ...shelved, tags: ["x"], updated_at: bookClock(5) }; + // The rename the plan was built from shows no tags, which the first book never had. + const books = servedFrom([ + ["GET", "/api/books", [FIRST_BOOK, shelved]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", shelved], + ["PATCH", "/api/books/1", renamed, 200, { name: "Renamed Book" }], + ["GET", "/api/books/1", renamed], + ["PATCH", "/api/books/2", retagged, 200, { tags: ["x"] }], + ["GET", "/api/books/2", retagged], + ]); + expect(books.state.writes.get("patch_books_by_id")).toMatchObject({ + served: true, + body: { kinds: [{ member: "tags", kind: "array" }] }, + }); + const before = books.snapshot(); + expect( + books.refusal("patch_books_by_id", { id: "1", body: { tags: "x" } }), + ).toMatchObject({ + message: expect.stringContaining("Member tags must be a JSON array"), + }); + expect(books.snapshot()).toBe(before); + books.answer("patch_books_by_id", { id: "1", body: { tags: ["y"] } }); + expect(entityOf(books, "1")?.fields).toMatchObject({ tags: ["y"] }); + }); + + it("generates the same values after a refused call as without it", () => { + const refusedFirst = servedFrom(); + refusedFirst.refusal("post_books", { body: {} }); + refusedFirst.refusal("patch_books_by_id", { + id: "1", + body: { title: "New" }, + }); + expect(refusedFirst.answer("post_books", CREATE_NEW_BOOK).body).toEqual( + servedFrom().answer("post_books", CREATE_NEW_BOOK).body, + ); + }); +}); + +/** Items whose `rank` is a collection sequence, with a recorded create at `recorded`. */ +const rankedItems = (initial: readonly [number, number], recorded: number) => { + const item = (id: number, rank: number) => ({ + id, + name: `n${String(id)}`, + rank, + }); + const [first, second] = initial; + return servedFrom([ + ["GET", "/api/items", [item(1, first), item(2, second)]], + ["GET", "/api/items/1", item(1, first)], + ["GET", "/api/items/2", item(2, second)], + ["POST", "/api/items", item(3, recorded), 201, { name: "n3" }], + ["GET", "/api/items/3", item(3, recorded)], + ["GET", "/api/items", [item(1, first), item(2, second), item(3, recorded)]], + ]); +}; + +describe("generated values", () => { + it("answers the same calls identically after a reset, never reusing a deleted id", () => { + const books = servedFrom(); + const session = () => { + const answers = [ + books.answer("post_books", CREATE_NEW_BOOK), + books.answer("patch_books_by_id", { + id: "3", + body: { name: "Newer Book" }, + }), + books.answer("delete_books_by_id", { id: "3" }), + books.answer("post_books", { body: { name: "Another Book" } }), + books.answer("get_books"), + ]; + return { answers, text: JSON.stringify([answers, books.snapshot()]) }; + }; + const first = session(); + books.store.reset(); + expect(session().text).toBe(first.text); + expect(first.answers[3]?.body).toMatchObject({ + id: 4, + name: "Another Book", + }); + }); + + it("serves the next rank when a collection sequence runs through negative values", () => { + const acrossZero = rankedItems([-2, -1], 0); + expect( + acrossZero.answer("post_items", { body: { name: "n4" } }).body, + ).toMatchObject({ id: 3, rank: 0 }); + expect( + acrossZero.answer("post_items", { body: { name: "n5" } }).body, + ).toMatchObject({ id: 4, rank: 1 }); + // The value after a stored negative rank stays on that sequence. + const negative = rankedItems([-4, -3], -2); + expect( + negative.answer("post_items", { body: { name: "n4" } }).body, + ).toMatchObject({ id: 3, rank: -2 }); + expect( + negative.answer("post_items", { body: { name: "n5" } }).body, + ).toMatchObject({ id: 4, rank: -1 }); + }); + + it("restarts the clock and the sequences when data/add loads a state", () => { + const books = servedFrom(); + const initial: unknown = JSON.parse(books.snapshot()); + const created = books.answer("post_books", CREATE_NEW_BOOK); + books.answer("post_books", { body: { name: "Another Book" } }); + expect(books.store.replace(initial).ok).toBe(true); + // Had the sequence and clock run on, this create would take id 5 one second later. + expect(books.answer("post_books", CREATE_NEW_BOOK).body).toEqual( + created.body, + ); + }); +}); + +describe("served catalog", () => { + it("adds only the served write tools, saying what each does to the state", () => { + const { source, state } = servedFrom(); + const catalog = servedCatalog(source, state, "books"); + if (!catalog.ok) throw catalog.error; + const described = new Map( + catalog.value.tools.map((tool) => [tool.name, tool]), + ); + // The export write targets no state type, so it is not served. + expect([...described.keys()].sort()).toEqual([ + "delete_books_by_id", + "get_books", + "get_books_by_id", + "patch_books_by_id", + "post_books", + ]); + expect(described.get("post_books")?.description).toContain( + 'creates an entity of /api/books in the served environment state (body members "name"; required: "name"), modelled on the recorded calls', + ); + expect(described.get("patch_books_by_id")?.description).toContain( + 'changes the entity the path names in the served environment state (body members "name"; send at least one)', + ); + expect(described.get("delete_books_by_id")?.description).toContain( + "removes the entity the path names from the served environment state", + ); + expect(described.get("post_books")?.annotations.readOnlyHint).toBe(false); + expect(described.get("get_books")?.description).toContain( + "answers from the served environment state where the state backs this tool", + ); + }); + + it("quotes the member names a write's description lists and samples a long one", () => { + const long = "n".repeat(90_000); + const item = (id: number, name: string) => ({ + id, + name, + "x, y": name, + [long]: name, + }); + const { source, state } = servedFrom([ + ["GET", "/api/books", [item(1, "A")]], + ["GET", "/api/books/1", item(1, "A")], + [ + "POST", + "/api/books", + item(2, "B"), + 201, + { name: "B", "x, y": "B", [long]: "B" }, + ], + ["GET", "/api/books/2", item(2, "B")], + ["GET", "/api/books", [item(1, "A"), item(2, "B")]], + ]); + const catalog = servedCatalog(source, state, "books"); + if (!catalog.ok) throw catalog.error; + const created = catalog.value.tools.find( + (tool) => tool.name === "post_books", + ); + expect(created?.description).toContain( + 'body members "name", "x, y", … (2/3 shown; full list in state Evidence)', + ); + }); + + it("serves every manifest tool unchanged in replay mode", () => { + const { source } = servedFrom(); + const replay = servedCatalog(source, undefined, "books"); + const definitions = environmentToolDefinitions(source, "books"); + if (!replay.ok || !definitions.ok) throw new Error("Expected a catalog"); + expect(replay.value).toEqual({ + tools: definitions.value, + dataPlane: "none", + }); + expect(replay.value.tools.map(({ name }) => name)).toContain( + "post_exports", + ); + }); +}); + +describe("paged list lines", () => { + it("answers an argument-less list from the state on a line that only pages, and another page from its recording", () => { + const paged = servedFrom([ + ["GET", "/api/books?limit=50", [FIRST_BOOK, SECOND_BOOK]], + ...BOOK_INIT_ROWS.slice(1), + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + ["GET", "/api/books/3", THIRD_BOOK], + ["GET", "/api/books?limit=50", [FIRST_BOOK, SECOND_BOOK, THIRD_BOOK]], + ]); + const created = paged.answer("post_books", CREATE_NEW_BOOK).body; + expect(paged.answer("get_books")).toMatchObject({ + body: [FIRST_BOOK, SECOND_BOOK, created], + replay: { record_ordinal: 0, exact_match: false }, + state: { source: "state", view: "list", listed: 3 }, + }); + expect(paged.answer("get_books", { limit: "10" })).toMatchObject({ + body: [FIRST_BOOK, SECOND_BOOK], + state: { source: "recording", reason: "unrecorded-request" }, + }); + }); +}); + +describe("path values", () => { + const orgBook = (id: number, org: string, name: string): Book => ({ + id, + org, + name, + }); + const ALPHA = orgBook(1, "my org1", "Alpha"); + const BETA = orgBook(2, "acme2", "Beta"); + const GAMMA = orgBook(3, "my org1", "Gamma"); + const DELTA = orgBook(4, "acme2", "Delta"); + /** Books under two organisations, one whose name the paths percent-encode, and a create under each. */ + const ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/orgs/my%20org1/books", [ALPHA]], + ["GET", "/api/orgs/acme2/books", [BETA]], + ["GET", "/api/orgs/my%20org1/books/1", ALPHA], + ["GET", "/api/orgs/acme2/books/2", BETA], + ["POST", "/api/orgs/my%20org1/books", GAMMA, 201, { name: "Gamma" }], + ["GET", "/api/orgs/my%20org1/books", [ALPHA, GAMMA]], + ["POST", "/api/orgs/acme2/books", DELTA, 201, { name: "Delta" }], + ["GET", "/api/orgs/acme2/books", [BETA, DELTA]], + ]; + + it("stores a parent path value decoded, as the gate learned it, whichever spelling the call uses", () => { + for (const spelling of ["my%20org1", "my org1"]) { + const books = servedFrom(ROWS); + const tool = books.source.manifest.tools.find( + (candidate) => candidate.method === "POST", + )!; + expect(books.state.writes.get(tool.name)).toMatchObject({ + served: true, + rules: expect.arrayContaining([ + expect.objectContaining({ field: "org", rule: "path" }), + ]), + }); + const [org] = environmentToolArguments(tool).path.values(); + books.answer(tool.name, { [org!]: spelling, body: { name: "Epsilon" } }); + const created = books.store + .snapshot() + .rea_state.entities.find( + (entity) => entity.fields["name"] === "Epsilon", + ); + expect(created).toMatchObject({ + collection: "/api/orgs/my%20org1/books", + fields: { org: "my org1" }, + }); + } + }); +}); + +describe("collections a create chooses", () => { + const orgBook = (id: number, name: string): Book => ({ id, name }); + /** One organisation listed under two spellings and another organisation, each with a create. */ + const ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/orgs/%61cme1/books", [orgBook(1, "Alpha")]], + ["GET", "/api/orgs/acme1/books", [orgBook(2, "Beta")]], + ["GET", "/api/orgs/beta2/books", [orgBook(3, "Gamma")]], + [ + "POST", + "/api/orgs/acme1/books", + orgBook(4, "Delta"), + 201, + { name: "Delta" }, + ], + ["GET", "/api/orgs/acme1/books", [orgBook(2, "Beta"), orgBook(4, "Delta")]], + [ + "POST", + "/api/orgs/beta2/books", + orgBook(5, "Epsilon"), + 201, + { name: "Epsilon" }, + ], + [ + "GET", + "/api/orgs/beta2/books", + [orgBook(3, "Gamma"), orgBook(5, "Epsilon")], + ], + ]; + + it("creates in the collection the call spells verbatim, as item calls choose their entity", () => { + const books = servedFrom(ROWS); + const tool = books.source.manifest.tools.find( + (candidate) => candidate.method === "POST", + )!; + expect(books.state.writes.get(tool.name)).toMatchObject({ served: true }); + const [org] = environmentToolArguments(tool).path.values(); + books.answer(tool.name, { [org!]: "acme1", body: { name: "Zeta" } }); + expect( + books.store + .snapshot() + .rea_state.entities.find((entity) => entity.fields["name"] === "Zeta") + ?.collection, + ).toBe("/api/orgs/acme1/books"); + }); + + it("refuses an empty parent segment before storing an unreadable collection", () => { + const books = servedFrom(ROWS); + const tool = books.source.manifest.tools.find( + (candidate) => candidate.method === "POST", + )!; + const [org] = environmentToolArguments(tool).path.values(); + const before = books.snapshot(); + expect( + books.refusal(tool.name, { [org!]: "", body: { name: "Invalid" } }), + ).toMatchObject({ + code: "invalid_request", + message: expect.stringContaining("path argument to be non-empty"), + }); + expect(books.snapshot()).toBe(before); + expect( + books.store.knownCollectionsOf("/api/orgs/{org}/books"), + ).not.toContain("/api/orgs//books"); + }); +}); + +describe("missing entities after a delete", () => { + it("answers with a recorded 2xx null, which the gate also took as an absence", () => { + const books = servedFrom([ + ...BOOK_INIT_ROWS, + ["DELETE", "/api/books/2", "", 204], + ["GET", "/api/books/2", null], + ]); + expect(books.state.writes.get("delete_books_by_id")).toMatchObject({ + served: true, + }); + books.answer("delete_books_by_id", { id: "1" }); + expect(books.answer("get_books_by_id", { id: "1" })).toMatchObject({ + status: 200, + body: null, + state: { view: "missing", id: "1" }, + }); + }); + + it("never answers a write of a missing entity with success, even when the write recorded a 2xx null", () => { + const books = servedFrom([ + ...BOOK_INIT_ROWS, + ["DELETE", "/api/books/2", null, 200], + ["GET", "/api/books/2", BOOK_NOT_FOUND, 404], + ]); + expect(books.state.writes.get("delete_books_by_id")).toMatchObject({ + served: true, + }); + expect(books.answer("delete_books_by_id", { id: "9" })).toMatchObject({ + status: 404, + body: BOOK_NOT_FOUND, + state: { view: "missing", id: "9" }, + }); + }); +}); + +describe("values the state started with", () => { + it("never reuses a seeded id for a create, even after its entity is deleted first", () => { + const books = servedFrom(); + books.answer("delete_books_by_id", { id: "2" }); + books.answer("post_books", CREATE_NEW_BOOK); + expect(entityOf(books, "2")).toBeUndefined(); + expect(entityOf(books, "3")?.fields).toMatchObject({ name: "New Book" }); + }); + + const orgBook = (id: number, org: string, name: string): Book => ({ + id, + org, + name, + }); + /** Books under two organisations, one spelled `first` in its paths, with creates and a delete the lists confirm. */ + const twoOrgs = (first: string, org: string) => { + const alpha = orgBook(1, org, "Alpha"); + const beta = orgBook(2, "acme2", "Beta"); + const gamma = orgBook(3, org, "Gamma"); + const delta = orgBook(4, "acme2", "Delta"); + const books = servedFrom([ + ["GET", `/api/orgs/${first}/books`, [alpha]], + ["GET", "/api/orgs/acme2/books", [beta]], + ["GET", `/api/orgs/${first}/books/1`, alpha], + ["GET", "/api/orgs/acme2/books/2", beta], + ["POST", `/api/orgs/${first}/books`, gamma, 201, { name: "Gamma" }], + ["GET", `/api/orgs/${first}/books`, [alpha, gamma]], + ["POST", "/api/orgs/acme2/books", delta, 201, { name: "Delta" }], + ["GET", "/api/orgs/acme2/books", [beta, delta]], + ["DELETE", `/api/orgs/${first}/books/3`, "", 204], + ["GET", `/api/orgs/${first}/books`, [alpha]], + ]); + const toolOf = (method: string) => + books.source.manifest.tools.find( + (candidate) => candidate.method === method, + )!; + const create = (value: string, name: string): string | undefined => { + const tool = toolOf("POST"); + const [parent] = environmentToolArguments(tool).path.values(); + books.answer(tool.name, { [parent!]: value, body: { name } }); + return books.store + .snapshot() + .rea_state.entities.find((entity) => entity.fields["name"] === name) + ?.collection; + }; + return { books, toolOf, create }; + }; + + it("creates in a collection's recorded spelling after the collection empties", () => { + const { books, toolOf, create } = twoOrgs("my%20org1", "my org1"); + const remove = toolOf("DELETE"); + const [parent, id] = environmentToolArguments(remove).path.values(); + for (const held of ["1", "3"]) + books.answer(remove.name, { [parent!]: "my%20org1", [id!]: held }); + expect(books.store.collectionsOf("/api/orgs/{org}/books")).not.toContain( + "/api/orgs/my%20org1/books", + ); + expect(create("my%20org1", "Epsilon")).toBe("/api/orgs/my%20org1/books"); + }); + + it("creates in the Evidence's spelling of a collection a loaded state does not hold", () => { + const { books, create } = twoOrgs("my%20org1", "my org1"); + const { rea_state: held } = books.store.snapshot(); + expect( + books.store.replace({ + rea_state: { + version: held.version, + entities: held.entities.filter( + (entity) => !entity.collection.includes("%20"), + ), + documents: held.documents, + }, + }).ok, + ).toBe(true); + expect(create("my%20org1", "Epsilon")).toBe("/api/orgs/my%20org1/books"); + }); + + it("creates in the loaded state's collection when the Evidence spells the same one differently", () => { + const { books, create } = twoOrgs("%61cme1", "acme1"); + const { rea_state: held } = books.store.snapshot(); + expect( + books.store.replace({ + rea_state: { + version: held.version, + entities: held.entities.map((entity) => + entity.collection.includes("%61cme1") + ? { + ...entity, + collection: entity.collection.replace("%61cme1", "a%63me1"), + record_ordinals: [], + } + : entity, + ), + documents: held.documents, + }, + }).ok, + ).toBe(true); + // Both spellings decode to the argument; the one holding books wins over the Evidence's emptied one. + expect(create("acme1", "Epsilon")).toBe("/api/orgs/a%63me1/books"); + }); + + it("creates in the spelling of a collection the capture listed empty", () => { + const beta = orgBook(2, "acme2", "Beta"); + const gamma = orgBook(3, "my org1", "Gamma"); + const delta = orgBook(4, "acme2", "Delta"); + // The state starts before the first create, when the encoded organisation's list was empty. + const books = servedFrom([ + ["GET", "/api/orgs/my%20org1/books", []], + ["GET", "/api/orgs/acme2/books", [beta]], + ["GET", "/api/orgs/acme2/books/2", beta], + ["POST", "/api/orgs/my%20org1/books", gamma, 201, { name: "Gamma" }], + ["GET", "/api/orgs/my%20org1/books", [gamma]], + ["POST", "/api/orgs/acme2/books", delta, 201, { name: "Delta" }], + ["GET", "/api/orgs/acme2/books", [beta, delta]], + ]); + const tool = books.source.manifest.tools.find( + (candidate) => candidate.method === "POST", + )!; + const [parent] = environmentToolArguments(tool).path.values(); + books.answer(tool.name, { + [parent!]: "my%20org1", + body: { name: "Epsilon" }, + }); + expect( + books.store + .snapshot() + .rea_state.entities.find( + (entity) => entity.fields["name"] === "Epsilon", + )?.collection, + ).toBe("/api/orgs/my%20org1/books"); + }); + + it("forgets the spellings of a loaded state once reset", () => { + const { books, create } = twoOrgs("%61cme1", "acme1"); + const { rea_state: held } = books.store.snapshot(); + // A loaded state spells the first organisation plainly; the Evidence's state spells it encoded. + expect( + books.store.replace({ + rea_state: { + version: held.version, + // Moved to another spelling, the entity cites none of the Evidence's records. + entities: held.entities.map((entity) => + entity.collection.includes("%61cme1") + ? { + ...entity, + collection: entity.collection.replace("%61cme1", "acme1"), + record_ordinals: [], + } + : entity, + ), + documents: held.documents, + }, + }).ok, + ).toBe(true); + books.store.reset(); + expect(create("acme1", "Epsilon")).toBe("/api/orgs/%61cme1/books"); + }); +}); + +describe("paged lines", () => { + it("answers a requested page the collection has outgrown from its recording, and a call with no query from the state", () => { + const books = servedFrom([ + ["GET", "/api/books?limit=2", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + // The application answers the same page with two books; serving would not answer it from the state. + ["GET", "/api/books?limit=2", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK, THIRD_BOOK]], + ]); + const created = books.answer("post_books", CREATE_NEW_BOOK).body as Book; + expect(books.answer("get_books", { limit: "2" })).toMatchObject({ + body: [FIRST_BOOK, SECOND_BOOK], + state: { source: "recording", reason: "page-exceeded" }, + }); + expect(books.answer("get_books")).toMatchObject({ + body: [FIRST_BOOK, SECOND_BOOK, created], + state: { source: "state", membership: "all", listed: 3 }, + }); + }); + + it("answers a whole list a zero limit names from the state, as zero lists every match", () => { + const books = servedFrom([ + ["GET", "/api/books?limit=0", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + ["GET", "/api/books/3", THIRD_BOOK], + ]); + const created = books.answer("post_books", CREATE_NEW_BOOK).body as Book; + expect(books.answer("get_books", { limit: "0" })).toMatchObject({ + body: [FIRST_BOOK, SECOND_BOOK, created], + state: { source: "state", membership: "all", listed: 3 }, + }); + }); + + it("answers a zero limit a whole line recorded empty from that recording once a create fills its collection", () => { + const alpha = { id: 1, name: "Alpha", org: "1" }; + const beta = { id: 2, name: "Beta", org: "2" }; + const books = servedFrom([ + ["GET", "/api/orgs/1/books", [alpha]], + ["GET", "/api/orgs/1/books/1", alpha], + ["GET", "/api/orgs/2/books?limit=0", []], + ["POST", "/api/orgs/2/books", beta, 201, { name: "Beta" }], + ["GET", "/api/orgs/2/books/2", beta], + ]); + books.answer("post_orgs_books_by_id", { id: "2", body: { name: "Gamma" } }); + // Only a recording that listed items under it shows that zero lists every entity. + expect( + books.answer("get_orgs_books_by_id", { id: "2", limit: "0" }), + ).toMatchObject({ + body: [], + state: { source: "recording", reason: "page-exceeded" }, + }); + }); + + it("keeps a range's bound beside a zero size its recording shows unbounded", () => { + const books = servedFrom([ + ["GET", "/api/books?limit=0&_start=0&_end=2", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + ["GET", "/api/books/3", THIRD_BOOK], + ]); + books.answer("post_books", CREATE_NEW_BOOK); + expect( + books.answer("get_books", { limit: "0", _start: "0", _end: "2" }), + ).toMatchObject({ + body: [FIRST_BOOK, SECOND_BOOK], + state: { source: "recording", reason: "page-exceeded" }, + }); + }); + + it("answers a range the collection has outgrown from its recording", () => { + const books = servedFrom([ + ["GET", "/api/books?_start=0&_end=2", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + ["GET", "/api/books?_start=0&_end=2", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/3", THIRD_BOOK], + ]); + books.answer("post_books", CREATE_NEW_BOOK); + expect(books.answer("get_books", { _start: "0", _end: "2" })).toMatchObject( + { + body: [FIRST_BOOK, SECOND_BOOK], + state: { source: "recording", reason: "page-exceeded" }, + }, + ); + }); +}); diff --git a/src/application/environment/environmentStateWriteAnswer.ts b/src/application/environment/environmentStateWriteAnswer.ts new file mode 100644 index 0000000000..bb40c3a4d2 --- /dev/null +++ b/src/application/environment/environmentStateWriteAnswer.ts @@ -0,0 +1,535 @@ +import { + createdFields, + documentBodyProblem, + updatedEntity, + updatedFields, + valueAt, + writtenDocument, + writtenResponse, + type GenerateValue, +} from "../../domain/analysisView/environmentStateWriteApply.js"; +import { + insertIndex, + nextInteger, +} from "../../domain/analysisView/environmentStateWriteExplain.js"; +import { + pathValuesOf, + type GeneratedRule, + type StateWritePlan, + type WriteCall, +} from "../../domain/analysisView/environmentStateWriteRules.js"; +import { + formatClock, + isUuid, + seededToken, + seededUuid, + spelledLike, +} from "../../domain/analysisView/environmentStateWriteValues.js"; +import { + isObject, + scalarText, + type JsonObject, + type StateEntity, +} from "../../domain/analysisView/environmentStateRules.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { + replayAnswer, + selectReplayExchange, + type ReplayArguments, +} from "./environmentReplay.js"; +import type { EnvironmentManifestTool } from "./environmentSource.js"; +import { + callSegments, + collectionMatch, + itemCall, + missingAnswer, + namedEntity, + recordedJson, + refused, + stateAnswer, + type StateAnswer, + type StateProvenance, + type StateServing, +} from "./environmentStateAnswer.js"; +import { createdCollection } from "./environmentStateCollections.js"; +import { + isLoadableValue, + unloadableMember, + type HeldDocument, +} from "./environmentStateDocument.js"; +import { environmentToolArguments } from "./environmentTools.js"; + +/** Attempts at a generated UUID or token before a collision with held values is refused. */ +const GENERATION_ATTEMPTS = 64; + +/** One served write call: the tool, its plan and its arguments. */ +interface WriteRequest { + readonly tool: EnvironmentManifestTool; + readonly plan: StateWritePlan; + readonly args: ReplayArguments; +} + +/** The path values a call supplies, keyed by template segment as write plans name them. */ +const pathValues = ( + tool: EnvironmentManifestTool, + args: ReplayArguments, +): Map => { + const names = environmentToolArguments(tool).path; + const template = tool.path_template + .slice(tool.path_template.indexOf("/") + 1) + .split("/"); + const values = new Map(); + for (const param of tool.params) { + const value = args[names.get(param.name) ?? param.name]; + const segment = template[param.index] ?? ""; + if (typeof value === "string") values.set(segment.slice(1, -1), value); + } + return values; +}; + +/** A refusal for a value `data/add` would refuse, so `data/get` always reports a state that loads back. */ +const unloadable = ( + tool: EnvironmentManifestTool, + holder: string, +): AnalysisError => + refused( + tool, + `${holder} would hold a value that is not exactly representable JSON, which data/add could not load back.`, + ); + +/** The call's body: an object, nothing, or a refusal for any other value or one holding a value the state cannot hold. */ +const callBody = ({ + tool, + args, +}: WriteRequest): Result => { + const name = environmentToolArguments(tool).body; + const value = name === undefined ? undefined : args[name]; + if (value === undefined) return ok(undefined); + if (!isObject(value as JsonValue)) + return err(refused(tool, `Tool ${tool.name} takes a JSON object body.`)); + const member = unloadableMember(value as JsonObject); + return member === undefined + ? ok(value as JsonObject) + : err(unloadable(tool, `Member ${member}`)); +}; + +/** The entity a recorded observation shows: the values a create keeps and an update's constants. */ +const observedEntity = ( + serving: StateServing, + plan: StateWritePlan, +): JsonObject | undefined => { + if (plan.observation === null) return undefined; + const exchange = serving.source.exchanges.get(plan.observation.ordinal); + const body = exchange === undefined ? undefined : recordedJson(exchange); + const object = + body === undefined ? undefined : valueAt(body, plan.observation.pointer); + return isObject(object) ? object : undefined; +}; + +/** One past the largest integer these entities hold in the field, or undefined when none holds one. */ +const nextHeld = (entities: readonly StateEntity[], field: string) => + nextInteger( + entities.map((entity) => entity.fields), + field, + ); + +/** + * Values one served write generates: its clock instant, sequences one past the largest held + * value, and UUIDs and tokens seeded by the state Evidence, tool, field and write count, + * skipping values the type already holds. + */ +const generator = ( + serving: StateServing, + request: WriteRequest, + target: { + readonly name: string; + readonly collection: readonly StateEntity[]; + readonly observed: JsonObject; + }, +): { + readonly generate: GenerateValue; + readonly taken: Map; +} => { + const { store } = serving; + const write = store.pendingWrite(); + const taken = new Map(); + const sequence = (key: string, held: number | undefined): number => { + const value = store.sequenceAfter(key, held); + taken.set(key, value); + return value; + }; + const ofType = store.ofType(request.plan.target ?? ""); + // A sequence never reuses a value the state started with, even after its entity was deleted. + const started = store.startedWith(request.plan.target ?? ""); + const startedIn = started.filter((held) => held.collection === target.name); + const unused = ( + field: string, + make: (attempt: number) => string, + ): Result => { + const held = new Set( + ofType.map((entity) => scalarText(entity.fields[field])), + ); + for (let attempt = 0; attempt < GENERATION_ATTEMPTS; attempt += 1) { + const value = make(attempt); + if (!held.has(value)) return ok(value); + } + return err( + `Could not generate a unique value for ${field} after ${String(GENERATION_ATTEMPTS)} attempts without colliding with the held state.`, + ); + }; + const generate = (rule: GeneratedRule): Result => { + const exemplar = target.observed[rule.field] ?? ""; + const seed = (attempt: number) => + `${store.state.evidenceId}:${request.tool.name}:${rule.field}:${String(write.count)}:${String(attempt)}`; + switch (rule.rule) { + case "clock": + return ok(formatClock(write.clockMs, rule.format)); + case "sequence": + return ok( + spelledLike( + exemplar, + sequence( + `${request.tool.name} ${rule.field} ${rule.scope === "collection" ? target.name : ""}`, + nextHeld( + rule.scope === "collection" + ? [...target.collection, ...startedIn] + : [...ofType, ...started], + rule.field, + ), + ), + ), + ); + case "unique": + return unused(rule.field, (attempt) => + seededUuid(seed(attempt), String(exemplar)), + ); + case "identity": + if (rule.generator === "sequence") + return ok( + spelledLike( + exemplar, + sequence( + `${request.plan.target ?? ""} ${rule.field} ${target.name}`, + nextHeld([...target.collection, ...startedIn], rule.field), + ), + ), + ); + return unused(rule.field, (attempt) => + rule.generator === "uuid" || isUuid(exemplar) + ? seededUuid(seed(attempt), String(exemplar)) + : seededToken(seed(attempt), String(exemplar)), + ); + } + }; + return { generate, taken }; +}; + +/** A write's answer on its recorded response's status and content type: a rendered body, none, or the recorded one. */ +const writeAnswer = ( + serving: StateServing, + request: WriteRequest, + rendered: { readonly fields: JsonObject | undefined; readonly id: string }, + state: StateProvenance, +): Result => { + const { plan, tool, args } = request; + const exchange = + plan.response === null + ? undefined + : serving.source.exchanges.get(plan.response.ordinal); + if (plan.response === null || exchange === undefined) + return err(refused(tool, `Tool ${tool.name} has no recorded response.`)); + const exact = + selectReplayExchange(tool, new Map([[exchange.ordinal, exchange]]), args) + ?.exact === true; + if (plan.response.shape === "recorded") + return ok({ ...replayAnswer(serving.source, exchange, exact), state }); + const body = writtenResponse(plan.response, recordedJson(exchange), rendered); + if (!body.ok) return err(refused(tool, body.error)); + return stateAnswer(serving, { exchange, exact }, body.value ?? null, state); +}; + +const answerCreate = ( + serving: StateServing, + request: WriteRequest, + call: WriteCall, +): Result => { + const { tool, plan } = request; + const type = plan.target ?? ""; + const idField = serving.store.type(type)?.id_field ?? ""; + const collection = createdCollection( + serving, + request.tool, + request.args, + plan.target ?? "", + ); + const observed = observedEntity(serving, plan); + if (collection === undefined || observed === undefined) + return err( + refused( + tool, + `Tool ${tool.name} needs every path argument to be non-empty.`, + ), + ); + if (plan.collection !== null && plan.collection !== collection) + return err( + refused( + tool, + `The recorded create's values are tied to ${plan.collection}; the environment models creates only there.`, + ), + ); + const members = serving.store.inCollection(type, collection); + const values = generator(serving, request, { + name: collection, + collection: members, + observed, + }); + // The gate learned path rules from the recorded path, decoded; the collection is this call's. + const created = { + ...call, + pathValues: pathValuesOf(tool.path_template, collection) ?? call.pathValues, + }; + const fields = createdFields(plan, observed, created, values.generate); + if (!fields.ok) return err(refused(tool, fields.error)); + const field = unloadableMember(fields.value); + if (field !== undefined) return err(unloadable(tool, `Field ${field}`)); + const id = scalarText(fields.value[idField]); + if ( + id === undefined || + serving.store.entity(type, collection, id) !== undefined + ) + return err( + refused( + tool, + `The state already holds a ${type} entity ${id ?? ""} in ${collection}.`, + ), + ); + const state = { + source: "state", + view: "create", + type, + collection, + id, + } as const; + const answer = writeAnswer( + serving, + request, + { fields: fields.value, id }, + state, + ); + if (!answer.ok) return answer; + const inserted = serving.store.insert( + { + type, + collection, + id, + fields: fields.value, + view_fields: [], + record_ordinals: [], + }, + insertIndex( + plan.insert ?? { rule: "last" }, + members.map((entity) => entity.fields), + fields.value, + ), + ); + if (!inserted.ok) return inserted; + serving.store.commitWrite(values.taken); + return answer; +}; + +/** The held entity an update or delete names, or the answer for one the state does not hold. */ +const targetEntity = ( + serving: StateServing, + request: WriteRequest, +): Result> => { + const { tool, plan, args } = request; + const call = itemCall(tool, args); + if (call === undefined) + return err( + err(refused(tool, `Tool ${tool.name} needs every path argument.`)), + ); + const entity = namedEntity(serving.store, plan.target ?? "", call); + return entity === undefined + ? err( + missingAnswer(serving, tool, { + type: plan.target ?? "", + collection: `/${call.segments.map(({ text }) => text).join("/")}`, + id: call.id, + }), + ) + : ok(entity); +}; + +const answerUpdate = ( + serving: StateServing, + request: WriteRequest, + call: WriteCall, +): Result => { + const { tool, plan } = request; + const target = targetEntity(serving, request); + if (!target.ok) return target.error; + const entity = target.value; + const observed = observedEntity(serving, plan) ?? {}; + const values = generator(serving, request, { + name: entity.collection, + collection: serving.store.inCollection(entity.type, entity.collection), + observed, + }); + const written = updatedFields( + plan, + { before: entity.fields, observed }, + { ...call, id: entity.id }, + values.generate, + ); + if (!written.ok) return err(refused(tool, written.error)); + // An increment can pass the exact integers. + const field = unloadableMember(written.value); + if (field !== undefined) return err(unloadable(tool, `Field ${field}`)); + const answer = writeAnswer( + serving, + request, + { fields: written.value, id: entity.id }, + { + source: "state", + view: "update", + type: entity.type, + collection: entity.collection, + id: entity.id, + }, + ); + if (!answer.ok) return answer; + const updated = serving.store.update(updatedEntity(entity, written.value)); + if (!updated.ok) return updated; + serving.store.commitWrite(values.taken); + return answer; +}; + +const answerDelete = ( + serving: StateServing, + request: WriteRequest, +): Result => { + const target = targetEntity(serving, request); + if (!target.ok) return target.error; + const entity = target.value; + const answer = writeAnswer( + serving, + request, + { fields: entity.fields, id: entity.id }, + { + source: "state", + view: "delete", + type: entity.type, + collection: entity.collection, + id: entity.id, + }, + ); + if (answer.ok) { + serving.store.remove(entity); + serving.store.commitWrite(new Map()); + } + return answer; +}; + +/** The held document a call changes: only at the path the state holds it, with members the capture sent as the JSON types the document holds. */ +const documentTarget = ( + serving: StateServing, + { tool, plan, args }: WriteRequest, + call: WriteCall, +): Result => { + const held = serving.store.document(plan.target ?? ""); + const segments = callSegments(tool, args); + if (held === undefined || plan.body === null) + return err( + refused(tool, `The state holds no document for ${plan.target ?? ""}.`), + ); + if (segments === undefined) + return err(refused(tool, `Tool ${tool.name} needs every path argument.`)); + // Document reads answer only on their recorded line, so a write elsewhere has no document to change. + if (collectionMatch(held.path, segments) === undefined) + return err( + refused( + tool, + `The state holds this document only at ${held.path}; the environment models writes to it only there.`, + ), + ); + const initial = serving.store.state.documents.find( + (document) => document.tool === held.tool, + ); + const problem = documentBodyProblem(plan, call, { + initial: initial?.value, + current: held.value, + }); + return problem === undefined ? ok(held) : err(refused(tool, problem)); +}; + +const answerDocument = ( + serving: StateServing, + request: WriteRequest, + call: WriteCall, +): Result => { + const { tool, plan } = request; + const target = documentTarget(serving, request, call); + if (!target.ok) return target; + const held = target.value; + const written = writtenDocument(plan, held.value, call); + if (!written.ok) return err(refused(tool, written.error)); + if (!isLoadableValue(written.value)) + return err(unloadable(tool, "The document")); + const state = { + source: "state", + view: "document-write", + path: held.path, + } as const; + const answer = + plan.response?.shape === "entity" + ? (() => { + const exchange = serving.source.exchanges.get(plan.response.ordinal); + return exchange === undefined + ? err(refused(tool, `Tool ${tool.name} has no recorded response.`)) + : stateAnswer( + serving, + { exchange, exact: true }, + written.value, + state, + ); + })() + : writeAnswer(serving, request, { fields: undefined, id: "" }, state); + if (!answer.ok) return answer; + const set = serving.store.setDocument(held.tool, written.value); + if (!set.ok) return set; + serving.store.commitWrite(new Map()); + return answer; +}; + +/** Apply one served write tool call to the live state and answer it; a refused call leaves the state unchanged. */ +export const answerWrite = ( + serving: StateServing, + request: WriteRequest, +): Result => { + const body = callBody(request); + if (!body.ok) return body; + const call: WriteCall = { + path: request.tool.path_template, + pathValues: pathValues(request.tool, request.args), + body: body.value, + }; + switch (request.plan.effect) { + case "create": + return answerCreate(serving, request, call); + case "update": + return answerUpdate(serving, request, call); + case "delete": + return answerDelete(serving, request); + case "document": + return answerDocument(serving, request, call); + case "none": + return err( + refused( + request.tool, + `Tool ${request.tool.name} is not served from the state.`, + ), + ); + } +}; diff --git a/src/application/environment/environmentStateWriteDescription.ts b/src/application/environment/environmentStateWriteDescription.ts new file mode 100644 index 0000000000..c3892fd41b --- /dev/null +++ b/src/application/environment/environmentStateWriteDescription.ts @@ -0,0 +1,47 @@ +import type { + StateWritePlan, + WriteBody, +} from "../../domain/analysisView/environmentStateWriteRules.js"; +import { describeList, quotedName } from "./environmentTools.js"; + +/** Captured member names, quoted and sampled as response fields are; the plan in the state Evidence holds every one. */ +const members = (names: readonly string[]): string => + describeList(names, names.length, quotedName, "state Evidence"); + +/** The body members a served write accepts, as a description clause. */ +const bodyClause = ( + body: WriteBody, + effect: StateWritePlan["effect"], +): string => { + const parts: string[] = []; + if (body.accepted.length > 0) + parts.push(`body members ${members(body.accepted)}`); + if (body.fixed.length > 0) + parts.push( + `${members(body.fixed.map(({ member }) => member))} only with the value the capture sent`, + ); + if (body.required.length > 0) + parts.push(`required: ${members(body.required)}`); + else if (effect === "update") parts.push("send at least one"); + return parts.length === 0 ? "" : ` (${parts.join("; ")})`; +}; + +/** What a served write tool does to the state, for its description; undefined for a tool the state does not serve. */ +export const describeWrite = ( + plan: StateWritePlan | undefined, +): string | undefined => { + if (plan?.served !== true) return undefined; + const body = plan.body === null ? "" : bodyClause(plan.body, plan.effect); + switch (plan.effect) { + case "create": + return `creates an entity of ${plan.target ?? ""} in the served environment state${body}, modelled on the recorded calls`; + case "update": + return `changes the entity the path names in the served environment state${body}, modelled on the recorded calls`; + case "delete": + return "removes the entity the path names from the served environment state, modelled on the recorded calls"; + case "document": + return `changes a document of the served environment state${body}, modelled on the recorded calls`; + case "none": + return undefined; + } +}; diff --git a/src/application/environment/environmentStateWriteRefusals.test.ts b/src/application/environment/environmentStateWriteRefusals.test.ts new file mode 100644 index 0000000000..c989c6777c --- /dev/null +++ b/src/application/environment/environmentStateWriteRefusals.test.ts @@ -0,0 +1,269 @@ +import { describe, expect, it } from "vitest"; + +import type { TimedCaptureRow } from "../../../tests/fixtures/analysisView.js"; +import { + servedFrom, + type Served, +} from "../../../tests/fixtures/environmentStateServing.js"; +import { + FIRST_BOOK, + SECOND_BOOK, + renamedBook, + type Book, +} from "../../../tests/fixtures/environmentStateWrites.js"; +import { MAX_JSON_DEPTH, type JsonValue } from "../../domain/jsonValue.js"; +import type { ReplayArguments } from "./environmentReplay.js"; +import { STATE_INTAKE_LIMIT_BYTES } from "./environmentStateDocument.js"; + +const DARK = { theme: "dark", meta: { n: 1 } }; +const LIGHT = { theme: "light", meta: { n: 2 } }; + +/** One organisation's settings document, read, patched with both members and read again. */ +const SETTINGS_ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/orgs/1/settings", DARK], + ["PATCH", "/api/orgs/1/settings", LIGHT, 200, LIGHT], + ["GET", "/api/orgs/1/settings", LIGHT], +]; + +const READ_SETTINGS = "get_orgs_settings_by_id"; +const PATCH_SETTINGS = "patch_orgs_settings_by_id"; + +/** Every refused call is a tool error naming its problem, and leaves the state as it was. */ +const expectRefusedUnchanged = ( + served: Served, + refused: readonly (readonly [string, ReplayArguments, string])[], +): void => { + const before = served.snapshot(); + for (const [name, args, message] of refused) { + expect(served.refusal(name, args)).toMatchObject({ + code: "invalid_request", + message: expect.stringContaining(message), + }); + expect(served.snapshot()).toBe(before); + } +}; + +/** What data/get reports loads back through data/add byte for byte. */ +const expectReloadsUnchanged = (served: Served): void => { + const before = served.snapshot(); + expect(served.store.replace(JSON.parse(before)).ok).toBe(true); + expect(served.snapshot()).toBe(before); +}; + +const nested = (depth: number): JsonValue => + Array.from({ length: depth }).reduce((inner) => [inner], 1); + +describe("served document writes", () => { + it("accepts only the members the capture sent, as the JSON types the document holds", () => { + const settings = servedFrom(SETTINGS_ROWS); + expect(settings.state.writes.get(PATCH_SETTINGS)).toMatchObject({ + served: true, + effect: "document", + }); + expectRefusedUnchanged(settings, [ + [ + PATCH_SETTINGS, + { id: "1", body: { theme: 5, extra: true } }, + "Member theme must be a JSON string", + ], + [ + PATCH_SETTINGS, + { id: "1", body: { extra: true } }, + "Member extra is not one the capture showed stored", + ], + ]); + const blue = { theme: "blue", meta: DARK.meta }; + expect( + settings.answer(PATCH_SETTINGS, { id: "1", body: { theme: "blue" } }), + ).toMatchObject({ + body: blue, + state: { view: "document-write", path: "/api/orgs/1/settings" }, + }); + expect(settings.answer(READ_SETTINGS, { id: "1" }).body).toEqual(blue); + }); + + it("leaves a document write unserved when its recording changed a member's JSON type, which serving would refuse", () => { + const retyped = { theme: 5, meta: LIGHT.meta }; + const settings = servedFrom([ + ["GET", "/api/orgs/1/settings", DARK], + ["PATCH", "/api/orgs/1/settings", retyped, 200, retyped], + ["GET", "/api/orgs/1/settings", retyped], + ]); + expect(settings.state.writes.get(PATCH_SETTINGS)).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }); + + it("does not confirm a document write at another path, which serving refuses", () => { + const settings = servedFrom([ + ["GET", "/api/orgs/1/settings", DARK], + ["PATCH", "/api/orgs/2/settings", LIGHT, 200, LIGHT], + ["GET", "/api/orgs/1/settings", LIGHT], + ]); + expect(settings.state.writes.get(PATCH_SETTINGS)).toMatchObject({ + served: false, + }); + }); + + it("changes the document only at the path the state holds it", () => { + const settings = servedFrom(SETTINGS_ROWS); + expectRefusedUnchanged(settings, [ + [ + PATCH_SETTINGS, + { id: "2", body: { theme: "blue" } }, + "holds this document only at /api/orgs/1/settings", + ], + ]); + expect(settings.answer(READ_SETTINGS, { id: "1" }).body).toEqual(DARK); + }); +}); + +describe("values the state can load back", () => { + it("refuses a body value data/add would refuse, whatever member carries it", () => { + const settings = servedFrom(SETTINGS_ROWS); + const unexact = + "Member meta would hold a value that is not exactly representable JSON"; + expectRefusedUnchanged(settings, [ + [PATCH_SETTINGS, { id: "1", body: { meta: { n: 1e20 } } }, unexact], + [ + PATCH_SETTINGS, + { + id: "1", + body: JSON.parse('{"meta":{"__proto__":{"n":1}}}') as JsonValue, + }, + unexact, + ], + [ + PATCH_SETTINGS, + { id: "1", body: { meta: { n: nested(MAX_JSON_DEPTH) } } }, + unexact, + ], + ]); + settings.answer(PATCH_SETTINGS, { id: "1", body: { meta: { n: 3 } } }); + expectReloadsUnchanged(settings); + }); + + it("refuses an increment past the exact integers, so data/get still loads back", () => { + const versioned = (entry: Book, version: number): Book => ({ + ...entry, + version, + }); + const last = Number.MAX_SAFE_INTEGER; + const edition = versioned(renamedBook(SECOND_BOOK, "Second Edition", 3), 2); + const books = servedFrom([ + [ + "GET", + "/api/books", + [versioned(FIRST_BOOK, last - 1), versioned(SECOND_BOOK, 1)], + ], + ["GET", "/api/books/1", versioned(FIRST_BOOK, last - 1)], + ["GET", "/api/books/2", versioned(SECOND_BOOK, 1)], + ["PATCH", "/api/books/2", edition, 200, { name: "Second Edition" }], + ["GET", "/api/books/2", edition], + ]); + const rename = (name: string) => ({ id: "1", body: { name } }); + expect( + books.answer("patch_books_by_id", rename("Renamed")).body, + ).toMatchObject({ version: last }); + expectRefusedUnchanged(books, [ + [ + "patch_books_by_id", + rename("Renamed Again"), + "Field version would hold a value that is not exactly representable JSON", + ], + ]); + expectReloadsUnchanged(books); + }); +}); + +describe("the state size limit", () => { + it("keeps the size data/get writes as each write changes the state", () => { + const books = servedFrom(); + const settings = servedFrom(SETTINGS_ROWS); + const tracked = (served: Served) => + expect(served.store.byteLength()).toBe( + Buffer.byteLength(served.snapshot(), "utf8"), + ); + tracked(books); + books.answer("post_books", { body: { name: "Ünïcode Bøok" } }); + tracked(books); + books.answer("patch_books_by_id", { id: "3", body: { name: "Short" } }); + tracked(books); + books.answer("delete_books_by_id", { id: "1" }); + books.answer("delete_books_by_id", { id: "2" }); + books.answer("delete_books_by_id", { id: "3" }); + tracked(books); + books.answer("post_books", { body: { name: "Only Book" } }); + tracked(books); + settings.answer(PATCH_SETTINGS, { id: "1", body: { theme: "dark blue" } }); + tracked(settings); + expect(settings.store.replace(JSON.parse(settings.snapshot())).ok).toBe( + true, + ); + tracked(settings); + }); + + it("refuses a create that would take data/get past what data/add loads", () => { + const books = servedFrom(); + // Each answer renders the book, so each stays under the reply body limit. + const create = { body: { name: "a".repeat(100_000) } }; + let refused: ReturnType | undefined; + for (let made = 0; made < 200 && refused === undefined; made += 1) { + const result = books.call("post_books", create); + if (!result.ok) refused = result; + } + const before = books.snapshot(); + expect(Buffer.byteLength(before, "utf8")).toBeLessThanOrEqual( + STATE_INTAKE_LIMIT_BYTES, + ); + expect(refused?.ok === false && refused.error).toMatchObject({ + resource: "capacity", + reportedLimits: { max_state_bytes: STATE_INTAKE_LIMIT_BYTES }, + }); + expect(books.refusal("post_books", create)).toMatchObject({ + code: "resource_constraint", + }); + expect(books.snapshot()).toBe(before); + expectReloadsUnchanged(books); + // A smaller state has room again. + books.answer("delete_books_by_id", { id: "3" }); + expect(books.answer("post_books", create).status).toBe(201); + }); + + it("refuses a document that data/get would write past what data/add loads", () => { + const books = servedFrom(); + const { rea_state: state } = books.store.snapshot(); + // Without the members data/get adds back, the document fits only until they return. + const document = { + rea_state: { + version: state.version, + entities: state.entities.map(({ type, collection, id, fields }) => ({ + type, + collection, + id, + fields: { ...fields }, + })), + documents: state.documents, + }, + }; + const first = document.rea_state.entities[0]!.fields; + first["name"] = `${String(first["name"])}${"a".repeat( + STATE_INTAKE_LIMIT_BYTES - + Buffer.byteLength(JSON.stringify(document), "utf8"), + )}`; + expect(Buffer.byteLength(JSON.stringify(document), "utf8")).toBe( + STATE_INTAKE_LIMIT_BYTES, + ); + const before = books.snapshot(); + const replaced = books.store.replace(document); + expect(replaced.ok).toBe(false); + expect(!replaced.ok && replaced.error.issues).toEqual([ + expect.objectContaining({ + reason: "out_of_range", + maximum: STATE_INTAKE_LIMIT_BYTES, + }), + ]); + expect(books.snapshot()).toBe(before); + }); +}); diff --git a/src/application/environment/environmentToolAnswer.ts b/src/application/environment/environmentToolAnswer.ts new file mode 100644 index 0000000000..5b2e4bf010 --- /dev/null +++ b/src/application/environment/environmentToolAnswer.ts @@ -0,0 +1,34 @@ +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import type { Result } from "../../domain/result.js"; +import { + replayEnvironmentTool, + type ReplayArguments, + type ReplayResult, +} from "./environmentReplay.js"; +import type { + EnvironmentManifestTool, + EnvironmentSource, +} from "./environmentSource.js"; +import { answerFromState } from "./environmentStateAnswer.js"; +import type { EnvironmentStateStore } from "./environmentStateStore.js"; +import { answerWrite } from "./environmentStateWriteAnswer.js"; + +/** How a served environment answers one tool call. */ +export type EnvironmentToolAnswer = ( + tool: EnvironmentManifestTool, + args: ReplayArguments, +) => Result; + +/** The answer a served environment gives: replay without a state store; with one, a served write applies to the live state and a read answers from it. */ +export const environmentToolAnswer = ( + source: EnvironmentSource, + store: EnvironmentStateStore | undefined, +): EnvironmentToolAnswer => + store === undefined + ? (tool, args) => replayEnvironmentTool(source, tool, args) + : (tool, args) => { + const plan = store.state.writes.get(tool.name); + return plan?.served === true + ? answerWrite({ source, store }, { tool, plan, args }) + : answerFromState({ source, store }, tool, args); + }; diff --git a/src/application/environment/environmentTools.ts b/src/application/environment/environmentTools.ts index 847be3d76d..9a0e4a2176 100644 --- a/src/application/environment/environmentTools.ts +++ b/src/application/environment/environmentTools.ts @@ -125,7 +125,7 @@ const describeText = ( }; /** Sample whole values before joining; keep full lists in the Evidence that owns them. */ -const describeList = ( +export const describeList = ( values: Iterable, count: number, display: (value: Value) => string = String, @@ -156,7 +156,7 @@ const describeList = ( * A captured name as a JSON string, so a separator, quote or newline in it cannot read as more of the * description. A name too long for a sample is left as it is: the sample refuses it either way. */ -const quotedName = (name: string): string => +export const quotedName = (name: string): string => name.length > DESCRIPTION_SAMPLE_BYTES ? name : JSON.stringify(name); /** Field names of the objects in first-seen order; the recorded bodies stay in the capture Evidence. */ @@ -225,6 +225,15 @@ const describeResponses = ( return `${returns} ${describeJson(found.value)}; ${recorded}`; }; +/** One tool's description: method, path, response summary, what it does, the records it cites and the Evidence. */ +const toolDescription = ( + source: EnvironmentSource, + tool: EnvironmentManifestTool, + describe: EnvironmentCatalogOptions["describe"], + answers: string, +): string => + `${REPLAY_CATALOG_INSTRUCTIONS} ${describeText(tool.method, "method")} ${describeText(tool.path_template, "path")}; ${describeResponses(tool, source.exchanges)}; ${describe?.(tool) ?? answers} [${describeList(tool.record_ordinals, tool.record_ordinals.length)}]; evidence ${source.evidenceId}`; + /** Captured argument names longer than this are advertised under a short alias; the manifest keeps the name. */ export const MAX_ARGUMENT_NAME_CHARS = 64; @@ -350,6 +359,8 @@ export type EnvironmentDataPlane = "none" | "state"; export interface EnvironmentCatalogOptions { readonly operation?: string; readonly dataPlane?: EnvironmentDataPlane; + /** What a served tool does, in place of how it answers; undefined keeps the answering sentence. */ + readonly describe?: (tool: EnvironmentManifestTool) => string | undefined; } /** Admit complete discovery before binding; six bytes per UTF-16 unit covers JSON escaping and UTF-8 expansion. */ @@ -359,6 +370,7 @@ export const environmentToolDefinitions = ( { operation = ENVIRONMENT_SERVE_OPERATION, dataPlane = "none", + describe, }: EnvironmentCatalogOptions = {}, ): Result< readonly EnvironmentToolDefinition[], @@ -414,7 +426,7 @@ export const environmentToolDefinitions = ( // A lower bound rejects enormous argument lists before constructing their alias map. if (!reserve(count * EMPTY_PROPERTY_BYTES)) return refused(); // The response summary is bounded by its samples and charged with the whole description. - const description = `${REPLAY_CATALOG_INSTRUCTIONS} ${describeText(tool.method, "method")} ${describeText(tool.path_template, "path")}; ${describeResponses(tool, source.exchanges)}; ${answers} [${describeList(tool.record_ordinals, tool.record_ordinals.length)}]; evidence ${source.evidenceId}`; + const description = toolDescription(source, tool, describe, answers); if (!text(description)) return refused(); const args = environmentToolArguments(tool); const properties = new Map(); diff --git a/src/cli/environmentCommands.ts b/src/cli/environmentCommands.ts index f84d4967a6..90ecdb0ba3 100644 --- a/src/cli/environmentCommands.ts +++ b/src/cli/environmentCommands.ts @@ -350,7 +350,7 @@ export const registerEnvironmentCommands = ( registerExportCommand(cli, logger); cli.command(CLI_COMMANDS.environmentServe, { description: - "Serve an environment manifest as an agent environment over streamable HTTP until SIGINT or SIGTERM: read tools answer from the state the capture observed, derived at startup, or every tool replays retained capture records with --replay-only or when the state is unavailable", + "Serve an environment manifest as an agent environment over streamable HTTP until SIGINT or SIGTERM: read tools, and write tools whose effect the capture confirmed, answer from the state the capture observed, derived at startup, or every tool replays retained capture records with --replay-only or when the state is unavailable", args: z.object({ manifest: manifestArgument }), options: z.object({ evidence: evidenceOption, diff --git a/src/contracts/analysisViewToolContracts.ts b/src/contracts/analysisViewToolContracts.ts index 4dc7042b17..84ddd26306 100644 --- a/src/contracts/analysisViewToolContracts.ts +++ b/src/contracts/analysisViewToolContracts.ts @@ -17,7 +17,7 @@ export const ANALYSIS_VIEW_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_analysis_view"), kind: "application", description: - 'Inspect selected views of completed Frida session status, GDB/objdump command observations, Go build metadata, Rust embedded dependencies, binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, decode_interface_builder, inspect_evm_interface, inspect_managed_artifact, inspect_managed_members, inspect_android_package, inspect_dart_aot, inspect_il2cpp_metadata, inspect_model_artifact, inspect_gpu_code_objects or inspect_web_network_capture Evidence without repeating analysis. Command summary views preserve ordered GDB records or opaque objdump stdout/stderr once, source parameters, completion facts and parent identity. They do not establish current target state. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. Go metadata uses kind go, collection dependencies/settings, exact paths/keys or indices, fields and pagination. Row index refers to the original collection; empty selections return no rows, and dependency coverage remains unknown. Rust metadata uses kind rust, collection packages/edges, exact names/versions or original indices, selected fields, edge from_indices/to_indices and matching-row pagination. Embedded-record coverage is separate from unknown runtime dependencies. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. The environment-state view extracts, from the same parents, the initial state the recorded reads observed: view {kind: "environment-state", template_rules: [...]} with the rules of the manifest it accompanies. It reads response bodies recorded before the first related write and reports entity types with an identity field (proved by a path parameter or inferred from a field name), entities in collection order with per-view values where views differ, documents for non-parameterized reads that are not entities and have one initial request line (item-shaped paths stay recorded), and a render plan per read tool: item and list views reproduce their latest recordings from the state, and every other read tool is a recorded view with a reason. Types and identity are inferences; values are recorded data, so it refuses parents inspected with sensitive values, and a scenario parent must have selected response bodies; its values keep the capture redaction markers, and a write whose URL it redacted ends the state of every type. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings and carry the source selection and coverage. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Protobuf views use kind protobuf, collection bodies/frames/fields, with record_ordinals, body_pointers and fields; field pages also select frame_indices, field_ids, field_numbers or schema_names. Byte identity, offset basis and body status remain present. Text and schema_value are opt-in page columns. Model views use kind model-artifact and collection metadata (keys), tensors (names/types) or types (derived totals), with indices, fields and paging. Header SHA-256 scopes declarations; full-file identity remains unknown unless full hashing was selected. Arrays retain typed eight-element previews with explicit omissions. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods/fields/properties/events/parameters/string_literals/code_gen_modules/method_addresses/return_types/field_types/parameter_types. Select indices, fields and pages; named rows accept names, types also accept image_indices/namespaces/declaring_types, and methods/members accept declaring_types. Parameters and parameter_types accept method_indices; code_gen_modules accept image_indices; method_addresses accept addresses/states, preserving every folded alternative. Row index is scoped to the metadata digest; native pages add binary_source. Uncollected native tables have null counts and remain unavailable. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native inventory Evidence from list_names, list/search_procedures or list/search_strings uses view: {kind: "native-inventory", offset: 0, limit: 64, fields: ["value"]}. Rows keep source_index and address even with selected fields. Summary counts describe retained rows; provider discovery coverage remains unknown. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', + 'Inspect selected views of completed Frida session status, GDB/objdump command observations, Go build metadata, Rust embedded dependencies, binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, decode_interface_builder, inspect_evm_interface, inspect_managed_artifact, inspect_managed_members, inspect_android_package, inspect_dart_aot, inspect_il2cpp_metadata, inspect_model_artifact, inspect_gpu_code_objects or inspect_web_network_capture Evidence without repeating analysis. Command summary views preserve ordered GDB records or opaque objdump stdout/stderr once, source parameters, completion facts and parent identity. They do not establish current target state. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. Go metadata uses kind go, collection dependencies/settings, exact paths/keys or indices, fields and pagination. Row index refers to the original collection; empty selections return no rows, and dependency coverage remains unknown. Rust metadata uses kind rust, collection packages/edges, exact names/versions or original indices, selected fields, edge from_indices/to_indices and matching-row pagination. Embedded-record coverage is separate from unknown runtime dependencies. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. The environment-state view extracts, from the same parents, the initial state the recorded reads observed: view {kind: "environment-state", template_rules: [...]} with the rules of the manifest it accompanies. It reads response bodies recorded before the first related write and reports entity types with an identity field (proved by a path parameter or inferred from a field name), entities in collection order with per-view values where views differ, documents for non-parameterized reads that are not entities and have one initial request line (item-shaped paths stay recorded), and a render plan per read tool: item and list views reproduce their latest recordings from the state, and every other read tool is a recorded view with a reason. It also reports a write plan per state-changing tool (create, update, delete or document, built from its first successful call) and write_clock; a plan is served only when replaying the capture shows a later read confirming its effect and none contradicting it. Types and identity are inferences; values are recorded data, so it refuses parents inspected with sensitive values, and a scenario parent must have selected response bodies; its values keep the capture redaction markers, and a write whose URL it redacted ends the state of every type. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings and carry the source selection and coverage. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Protobuf views use kind protobuf, collection bodies/frames/fields, with record_ordinals, body_pointers and fields; field pages also select frame_indices, field_ids, field_numbers or schema_names. Byte identity, offset basis and body status remain present. Text and schema_value are opt-in page columns. Model views use kind model-artifact and collection metadata (keys), tensors (names/types) or types (derived totals), with indices, fields and paging. Header SHA-256 scopes declarations; full-file identity remains unknown unless full hashing was selected. Arrays retain typed eight-element previews with explicit omissions. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods/fields/properties/events/parameters/string_literals/code_gen_modules/method_addresses/return_types/field_types/parameter_types. Select indices, fields and pages; named rows accept names, types also accept image_indices/namespaces/declaring_types, and methods/members accept declaring_types. Parameters and parameter_types accept method_indices; code_gen_modules accept image_indices; method_addresses accept addresses/states, preserving every folded alternative. Row index is scoped to the metadata digest; native pages add binary_source. Uncollected native tables have null counts and remain unavailable. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native inventory Evidence from list_names, list/search_procedures or list/search_strings uses view: {kind: "native-inventory", offset: 0, limit: 64, fields: ["value"]}. Rows keep source_index and address even with selected fields. Summary counts describe retained rows; provider discovery coverage remains unknown. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', inputSchema: inspectAnalysisViewInputSchema, outputSchema: evidenceResultOf(analysisViewResultSchema), examples: [ diff --git a/src/domain/analysisView/EnvironmentStateAccumulator.ts b/src/domain/analysisView/EnvironmentStateAccumulator.ts index 986217cce2..fdb9d001f4 100644 --- a/src/domain/analysisView/EnvironmentStateAccumulator.ts +++ b/src/domain/analysisView/EnvironmentStateAccumulator.ts @@ -46,6 +46,8 @@ type StateFields = Pick< | "entities" | "documents" | "views" + | "writes" + | "write_clock" | "unknowns" | "coverage" | "limitations" @@ -91,6 +93,17 @@ const redactedWriteSchema = z.object({ const contentSchema = z.object({ response: z.object({ content: z.object({ text: z.string() }) }), }); +const requestSchema = z.object({ + request: z.object({ + postData: z.object({ mimeType: z.string(), text: z.string() }), + }), +}); +const responseSizeSchema = z.object({ + response: z.object({ + status: z.number(), + content: z.object({ text: z.string().optional() }).optional(), + }), +}); interface RecordEntry { readonly facts: StateRecordFacts; @@ -115,7 +128,7 @@ const UNKNOWNS: StateView["unknowns"] = [ { kind: "state-transition", statement: - "The state is one snapshot from before the first related recorded write; it states no effects of writes.", + "The state is one snapshot from before the first related recorded write; write effects are modelled only from the calls the capture recorded, and served only where its later reads confirmed them.", }, ]; @@ -241,6 +254,43 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator return value === undefined ? { kind: "unusable" } : { kind: "json", value }; } + /** Parse one recorded request body as state JSON, as response bodies are admitted. */ + #requestBody(ordinal: number): ParsedBody | null { + const entry = this.#entries.get(ordinal); + const request = requestSchema.safeParse(entry?.record.reported); + if ( + entry === undefined || + !request.success || + !isJsonLike(mediaTypeEssence(request.data.request.postData.mimeType)) + ) + return { kind: "unusable" }; + const { mimeType, text } = request.data.request.postData; + if (Buffer.byteLength(text, "utf8") > STATE_BODY_LIMIT_BYTES) + return { kind: "unusable" }; + if (!this.temporary(RECORD_SCRATCH_BYTES + textBytes(text.length) * 4)) + return null; + try { + const value = stateBodyJson(mimeType, { bodyText: text }); + return value === undefined + ? { kind: "unusable" } + : { kind: "json", value }; + } finally { + this.releaseTemporary(); + } + } + + /** Whether a record's response carried no body: a 204, or empty content. */ + #emptyResponse(ordinal: number): boolean { + const parsed = responseSizeSchema.safeParse( + this.#entries.get(ordinal)?.record.reported, + ); + return ( + parsed.success && + (parsed.data.response.status === 204 || + (parsed.data.response.content?.text ?? "") === "") + ); + } + /** Project the init state of the manifest's primary origin; records keep timeline order. */ finishState(): Result { const manifest = this.finish(); @@ -274,6 +324,8 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator timeline, budget: { body: (ordinal) => this.#body(ordinal), + requestBody: (ordinal) => this.#requestBody(ordinal), + emptyResponse: (ordinal) => this.#emptyResponse(ordinal), retain: (bytes) => this.retain(bytes), release: () => this.releaseTemporary(), }, diff --git a/src/domain/analysisView/EnvironmentStateExtraction.ts b/src/domain/analysisView/EnvironmentStateExtraction.ts index c3c2168011..610ef6e196 100644 --- a/src/domain/analysisView/EnvironmentStateExtraction.ts +++ b/src/domain/analysisView/EnvironmentStateExtraction.ts @@ -54,6 +54,7 @@ import { type StatePlan, } from "./environmentStateGate.js"; import { StateWriteCutoffs } from "./environmentStateWriteCutoffs.js"; +import { extractWrites } from "./environmentStateWrites.js"; class StateBudgetExhausted extends Error {} @@ -74,6 +75,7 @@ const requestLine = (record: StateRecordFacts): string => /** Derive the init state and render plans from manifest tools and timeline-ordered records. */ export class EnvironmentStateExtraction { readonly #tools: readonly StateToolFacts[]; + readonly #records: readonly StateRecordFacts[]; readonly #timeline: StateTimeline; readonly #budget: StateBudget; readonly #position: ReadonlyMap; @@ -104,6 +106,7 @@ export class EnvironmentStateExtraction { readonly budget: StateBudget; }) { this.#tools = input.tools; + this.#records = input.records; this.#timeline = input.timeline; this.#budget = input.budget; this.#position = new Map( @@ -146,13 +149,38 @@ export class EnvironmentStateExtraction { plan.tool.name, plan.item ? checkItemPlan(gate, plan) : checkListPlan(gate, plan), ); - return this.#result(this.#documents(candidates)); + return this.#withWrites(this.#result(this.#documents(candidates))); } catch (cause: unknown) { if (cause instanceof StateBudgetExhausted) return null; throw cause; } } + /** The write plans the capture's own timeline confirms against the extracted state. */ + #withWrites( + state: Omit, + ): ExtractedState { + return { + ...state, + ...extractWrites(state, { + tools: this.#tools, + records: this.#records, + timed: this.#timeline === "recorded-start-time", + volatileOf: (type) => this.#volatileOf(type), + io: { + withBody: (ordinal, use) => this.#withBody(ordinal, use), + requestBody: (ordinal) => { + const parsed = this.#budget.requestBody(ordinal); + if (parsed === null) throw new StateBudgetExhausted(); + return parsed.kind === "json" ? parsed.value : undefined; + }, + emptyResponse: (ordinal) => this.#budget.emptyResponse(ordinal), + retain: (bytes) => this.#retain(bytes), + }, + }), + }; + } + #retain(bytes: number): void { if (!this.#budget.retain(bytes)) throw new StateBudgetExhausted(); } @@ -583,7 +611,9 @@ export class EnvironmentStateExtraction { } /** Types and entities that at least one state-backed view renders; view values only for those views. */ - #result(documents: StateDocument[]): ExtractedState { + #result( + documents: StateDocument[], + ): Omit { const backed = new Set(); const backedTypes = new Set(); for (const view of this.#views.values()) diff --git a/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts b/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts new file mode 100644 index 0000000000..49d27e7198 --- /dev/null +++ b/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts @@ -0,0 +1,2333 @@ +import { describe, expect, it } from "vitest"; + +import { + timedCaptureEvidence, + type TimedCaptureRow, +} from "../../../tests/fixtures/analysisView.js"; +import { + BOOK_INIT_ROWS, + BOOK_NOT_FOUND, + FIRST_BOOK, + SECOND_BOOK, + THIRD_BOOK, + book, + bookClock, + environmentStateViewOf, + renamedBook, + type Book, +} from "../../../tests/fixtures/environmentStateWrites.js"; +import type { JsonValue } from "../jsonValue.js"; +import { ok } from "../result.js"; +import { itemTarget, SimulatedState } from "./environmentStateWriteChecks.js"; +import { + laterObservation, + type ObservationSearch, +} from "./environmentStateWriteObserve.js"; +import { updatedFields } from "./environmentStateWriteApply.js"; +import { + retainFixed, + type EffectContext, +} from "./environmentStateWriteEffects.js"; + +/** The plan the gate gives `tool` over a capture of `rows`. */ +const gated = (rows: readonly TimedCaptureRow[], tool: string) => { + const { writes } = environmentStateViewOf(timedCaptureEvidence(rows)); + const plan = writes.find((candidate) => candidate.tool === tool); + if (plan === undefined) throw new Error(`The gate planned no ${tool}`); + return plan; +}; + +const RENAMED = renamedBook(FIRST_BOOK, "Renamed Book", 3); + +describe("reads compared as the served environment answers them", () => { + /** Item reads in a `{data, revision}` envelope, a rename, then a read recorded at `revision`. */ + const renamedInEnvelope = (revision: string): TimedCaptureRow[] => [ + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", { data: FIRST_BOOK, revision: "r1" }], + ["GET", "/api/books/2", { data: SECOND_BOOK, revision: "r1" }], + ["PATCH", "/api/books/1", RENAMED, 200, { name: "Renamed Book" }], + ["GET", "/api/books/1", { data: RENAMED, revision }], + ]; + + it("contradicts a write when an item read's envelope differs from the one served", () => { + expect(gated(renamedInEnvelope("r2"), "patch_books_by_id")).toMatchObject({ + served: false, + reason: "contradicted", + contradicted_by: 4, + }); + expect(gated(renamedInEnvelope("r1"), "patch_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [4], + }); + }); + + /** A list in a `{items, revision}` envelope, a create, then the list recorded at `revision`. */ + const createdInEnvelope = (revision: string): TimedCaptureRow[] => [ + ["GET", "/api/books", { items: [FIRST_BOOK, SECOND_BOOK], revision: "r1" }], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + [ + "GET", + "/api/books", + { items: [FIRST_BOOK, SECOND_BOOK, THIRD_BOOK], revision }, + ], + ]; + + it("contradicts a write when a list's envelope differs from its served line", () => { + expect(gated(createdInEnvelope("r2"), "post_books")).toMatchObject({ + served: false, + reason: "contradicted", + contradicted_by: 4, + }); + expect(gated(createdInEnvelope("r1"), "post_books")).toMatchObject({ + served: true, + checked_record_ordinals: [4], + }); + }); + + /** A filtered list of both books, a delete of the second, then the filtered list recorded with `listed`. */ + const deletedFromFiltered = (listed: readonly Book[]): TimedCaptureRow[] => [ + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + ["GET", "/api/books?shelf=a", [FIRST_BOOK, SECOND_BOOK]], + ["DELETE", "/api/books/2", "", 204], + ["GET", "/api/books?shelf=a", [...listed]], + ]; + + it("contradicts a delete that a filtered list still shows, and confirms one it drops", () => { + expect( + gated( + deletedFromFiltered([FIRST_BOOK, SECOND_BOOK]), + "delete_books_by_id", + ), + ).toMatchObject({ + served: false, + reason: "contradicted", + contradicted_by: 4, + }); + expect( + gated(deletedFromFiltered([FIRST_BOOK]), "delete_books_by_id"), + ).toMatchObject({ served: true, checked_record_ordinals: [4] }); + }); + + /** A delete confirmed by an item read, then the filtered list recorded with its query reordered. */ + const deletedThenReordered = (listed: readonly Book[]): TimedCaptureRow[] => [ + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + ["GET", "/api/books?shelf=a&sort=id", [FIRST_BOOK, SECOND_BOOK]], + ["DELETE", "/api/books/2", "", 204], + ["GET", "/api/books/2", BOOK_NOT_FOUND, 404], + ["GET", "/api/books?sort=id&shelf=a", [...listed]], + ]; + + it("checks a list read on a checked line however its query is ordered, as serving matches it", () => { + // The item read already checked the delete, so a list still showing the book marks it a side effect. + expect( + gated( + deletedThenReordered([FIRST_BOOK, SECOND_BOOK]), + "delete_books_by_id", + ), + ).toMatchObject({ served: false, reason: "side-effect" }); + expect( + gated(deletedThenReordered([FIRST_BOOK]), "delete_books_by_id"), + ).toMatchObject({ served: true, checked_record_ordinals: [4] }); + }); +}); + +describe("update constants", () => { + const paper = (entry: Book, format = "PAPER"): Book => ({ ...entry, format }); + const RENAMED_PAPER = paper(RENAMED); + const HARDCOVER = paper( + { ...SECOND_BOOK, updated_at: bookClock(5) }, + "HARDCOVER", + ); + /** A rename that keeps the format, then a format change the application stores in capitals. */ + const ROWS: TimedCaptureRow[] = [ + ["GET", "/api/books", [paper(FIRST_BOOK), paper(SECOND_BOOK)]], + ["GET", "/api/books/1", paper(FIRST_BOOK)], + ["GET", "/api/books/2", paper(SECOND_BOOK)], + ["PATCH", "/api/books/1", RENAMED_PAPER, 200, { name: "Renamed Book" }], + ["GET", "/api/books/1", RENAMED_PAPER], + ["PATCH", "/api/books/2", HARDCOVER, 200, { format: "hardcover" }], + ["GET", "/api/books/2", HARDCOVER], + ]; + + it("stores the value the call that showed a constant recorded, as serving applies the plan", () => { + const plan = gated(ROWS, "patch_books_by_id"); + expect(plan).toMatchObject({ + served: true, + observation: { ordinal: 3, pointer: "" }, + body: { fixed: [{ member: "format", sent: "hardcover" }] }, + checked_record_ordinals: [4, 6], + }); + // Serving reads the plan's observation, the first update's answer, which still shows PAPER. + const served = updatedFields( + plan, + { before: paper(SECOND_BOOK), observed: RENAMED_PAPER }, + { + path: "/api/books/2", + pathValues: new Map([["id", "2"]]), + body: { format: "hardcover" }, + id: "2", + }, + () => ok(bookClock(9)), + ); + expect(served).toEqual( + ok(paper({ ...SECOND_BOOK, updated_at: bookClock(9) }, "HARDCOVER")), + ); + }); +}); + +describe("write examples", () => { + it("takes no modelled effect or exemplar from a redirected write", () => { + const rows: TimedCaptureRow[] = [ + ...BOOK_INIT_ROWS, + ["DELETE", "/api/books/2", "", 302], + ["GET", "/api/books/2", SECOND_BOOK], + ["DELETE", "/api/books/2", "", 204], + ["GET", "/api/books/2", BOOK_NOT_FOUND, 404], + ]; + expect(gated(rows, "delete_books_by_id")).toMatchObject({ + served: true, + exemplar_ordinal: 5, + checked_record_ordinals: [6], + contradicted_by: null, + }); + }); +}); + +describe("update rules learned from a later call", () => { + const versioned = (entry: Book, version: number): Book => ({ + ...entry, + version, + }); + /** Two renames, of the first and the second book, each answered at the version given. */ + const renames = (first: number, second: number): TimedCaptureRow[] => { + const firstAfter = versioned( + renamedBook(FIRST_BOOK, "Renamed Book", 3), + first, + ); + const secondAfter = versioned( + renamedBook(SECOND_BOOK, "Second Edition", 5), + second, + ); + return [ + [ + "GET", + "/api/books", + [versioned(FIRST_BOOK, 1), versioned(SECOND_BOOK, 1)], + ], + ["GET", "/api/books/1", versioned(FIRST_BOOK, 1)], + ["GET", "/api/books/2", versioned(SECOND_BOOK, 1)], + ["PATCH", "/api/books/1", firstAfter, 200, { name: "Renamed Book" }], + ["GET", "/api/books/1", firstAfter], + ["PATCH", "/api/books/2", secondAfter, 200, { name: "Second Edition" }], + ["GET", "/api/books/2", secondAfter], + ]; + }; + + it("leaves an update unserved when one call increments a field another left as it was", () => { + for (const [first, second] of [ + [1, 2], + [2, 1], + ] as const) + expect(gated(renames(first, second), "patch_books_by_id")).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + expect(gated(renames(2, 2), "patch_books_by_id")).toMatchObject({ + served: true, + rules: expect.arrayContaining([{ field: "version", rule: "increment" }]), + checked_record_ordinals: [4, 6], + }); + }); +}); + +describe("slug rules learned from a later call", () => { + /** A patch of the first book sending `name`, answered with `answered`, then a rename of the second. */ + const renames = (name: string, answered: Book): TimedCaptureRow[] => { + const second = renamedBook(SECOND_BOOK, "Second Edition", 5); + return [ + ...BOOK_INIT_ROWS, + ["PATCH", "/api/books/1", answered, 200, { name }], + ["GET", "/api/books/1", answered], + ["PATCH", "/api/books/2", second, 200, { name: "Second Edition" }], + ["GET", "/api/books/2", second], + ]; + }; + + it("serves a slug rule an earlier call agrees with, and not one it contradicts", () => { + const kept = { ...FIRST_BOOK, updated_at: bookClock(3) }; + expect( + gated(renames("First Book", kept), "patch_books_by_id"), + ).toMatchObject({ + served: true, + rules: expect.arrayContaining([ + { field: "slug", rule: "slug", from: "name" }, + ]), + }); + // The application renamed the first book without following its slug. + expect( + gated( + renames("Renamed Book", { ...kept, name: "Renamed Book" }), + "patch_books_by_id", + ), + ).toMatchObject({ served: false, reason: "unexplained-change" }); + }); +}); + +describe("clock rules learned across calls", () => { + it("serves a clock only when every recorded call changed that field", () => { + const oldFirst = "2026-10-01T09:00:00Z"; + const oldSecond = "2026-10-01T10:00:00Z"; + const described = ( + entry: Book, + description: string, + publishedAt: string, + ): Book => ({ ...entry, description, published_at: publishedAt }); + const first = described(FIRST_BOOK, "original", oldFirst); + const second = described(SECOND_BOOK, "other", oldSecond); + const published = described( + renamedBook(FIRST_BOOK, "Renamed Book", 3), + "original", + bookClock(3), + ); + /** The first call stamps `published_at`; `later` is the second call's answer. */ + const rows = (later: Book, sent: Book): TimedCaptureRow[] => [ + ["GET", "/api/books", [first, second]], + ["GET", "/api/books/1", first], + ["GET", "/api/books/2", second], + ["PATCH", "/api/books/1", published, 200, { name: "Renamed Book" }], + ["GET", "/api/books/1", published], + ["PATCH", "/api/books/2", later, 200, sent], + ["GET", "/api/books/2", later], + ]; + // The description edit stamps `updated_at` and leaves `published_at` as it was. + const edited = described( + { ...SECOND_BOOK, updated_at: bookClock(5) }, + "revised", + oldSecond, + ); + expect( + gated(rows(edited, { description: "revised" }), "patch_books_by_id"), + ).toMatchObject({ served: false, reason: "unexplained-change" }); + const stamped = described( + renamedBook(SECOND_BOOK, "Second Edition", 5), + "other", + bookClock(5), + ); + const plan = gated( + rows(stamped, { name: "Second Edition" }), + "patch_books_by_id", + ); + expect(plan).toMatchObject({ + served: true, + checked_record_ordinals: [4, 6], + }); + expect(plan.rules).toContainEqual({ + field: "published_at", + rule: "clock", + format: { kind: "iso", separator: "T", fraction: 0, zone: "Z" }, + }); + }); + + it("leaves a clock unserved when an earlier call kept the field a later call stamped", () => { + const oldFirst = "2026-10-01T09:00:00Z"; + const oldSecond = "2026-10-01T10:00:00Z"; + const described = ( + entry: Book, + description: string, + publishedAt: string, + ): Book => ({ ...entry, description, published_at: publishedAt }); + const first = described(FIRST_BOOK, "original", oldFirst); + const second = described(SECOND_BOOK, "other", oldSecond); + const edited = described( + { ...SECOND_BOOK, updated_at: bookClock(3) }, + "revised", + oldSecond, + ); + const published = described( + renamedBook(FIRST_BOOK, "Renamed Book", 5), + "original", + bookClock(5), + ); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [first, second]], + ["GET", "/api/books/1", first], + ["GET", "/api/books/2", second], + ["PATCH", "/api/books/2", edited, 200, { description: "revised" }], + ["GET", "/api/books/2", edited], + ["PATCH", "/api/books/1", published, 200, { name: "Renamed Book" }], + ["GET", "/api/books/1", published], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }); +}); + +describe("checked lines", () => { + /** A list page that echoes its own request path, as some APIs do. */ + const page = (path: string, items: JsonValue[]): JsonValue => ({ + path, + items, + }); + const PLAIN = "/api/orgs/acme1/books"; + const ENCODED = "/api/orgs/%61cme1/books"; + const OTHER = "/api/orgs/beta2/books"; + /** Three organisations' books under one templated tool, two spelling one organisation differently, then deletes each confirmed by its list. */ + const ROWS: TimedCaptureRow[] = [ + ["GET", PLAIN, page(PLAIN, [FIRST_BOOK])], + ["GET", ENCODED, page(ENCODED, [SECOND_BOOK])], + ["GET", OTHER, page(OTHER, [THIRD_BOOK])], + ["GET", `${PLAIN}/1`, FIRST_BOOK], + ["GET", `${ENCODED}/2`, SECOND_BOOK], + ["GET", `${OTHER}/3`, THIRD_BOOK], + ["DELETE", `${ENCODED}/2`, "", 204], + ["GET", ENCODED, page(ENCODED, [])], + ["DELETE", `${OTHER}/3`, "", 204], + ["GET", OTHER, page(OTHER, [])], + ]; + + it("keeps a line's path spelling, as replay prefers the verbatim one", () => { + const view = environmentStateViewOf(timedCaptureEvidence(ROWS), [ + "sibling-variance", + ]); + expect(view.views).toContainEqual( + expect.objectContaining({ + shape: "list", + checked_record_ordinals: [0, 1, 2], + }), + ); + expect( + view.writes.find((candidate) => candidate.effect === "delete"), + ).toMatchObject({ served: true, checked_record_ordinals: [7, 9] }); + }); + /** Organisations listed only under encoded or other spellings, a delete in each, then a list of the first spelled plainly. */ + const deletedThenPlain = (listed: readonly Book[]): TimedCaptureRow[] => { + const third = "/api/orgs/gamma3/books"; + const fourth = book(4, "Fourth Book", "2026-10-01T11:00:00Z"); + return [ + ["GET", ENCODED, [SECOND_BOOK]], + ["GET", OTHER, [THIRD_BOOK]], + ["GET", third, [fourth]], + ["GET", `${ENCODED}/2`, SECOND_BOOK], + ["GET", `${OTHER}/3`, THIRD_BOOK], + ["DELETE", `${ENCODED}/2`, "", 204], + ["GET", `${ENCODED}/2`, BOOK_NOT_FOUND, 404], + ["DELETE", `${OTHER}/3`, "", 204], + ["GET", `${OTHER}/3`, BOOK_NOT_FOUND, 404], + ["GET", PLAIN, [...listed]], + ]; + }; + + it("checks a read spelled plainly against the encoded line serving answers it from", () => { + const deleteOf = (listed: readonly Book[]) => + environmentStateViewOf(timedCaptureEvidence(deletedThenPlain(listed)), [ + "sibling-variance", + ]).writes.find((candidate) => candidate.effect === "delete"); + // Serving lists the encoded collection for the plain argument, which no longer holds the book. + expect(deleteOf([SECOND_BOOK])).toMatchObject({ + served: false, + reason: "contradicted", + contradicted_by: 9, + }); + expect(deleteOf([])).toMatchObject({ + served: true, + checked_record_ordinals: [6, 8, 9], + }); + }); +}); + +describe("insert rules", () => { + it("still learns where a list places a create after an item read confirmed it", () => { + const alpha = book(3, "Alpha", bookClock(3)); + const rows: TimedCaptureRow[] = [ + ...BOOK_INIT_ROWS, + ["POST", "/api/books", alpha, 201, { name: "Alpha" }], + ["GET", "/api/books/3", alpha], + ["GET", "/api/books", [alpha, FIRST_BOOK, SECOND_BOOK]], + ]; + expect(gated(rows, "post_books")).toMatchObject({ + served: true, + insert: { rule: "sorted", field: "name", direction: "ascending" }, + checked_record_ordinals: [4, 5], + }); + }); +}); + +describe("reads the served answer no longer matches", () => { + it("contradicts an update when a later read's body no longer holds the entity", () => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", { data: FIRST_BOOK }], + ["GET", "/api/books/2", { data: SECOND_BOOK }], + ["PATCH", "/api/books/1", RENAMED, 200, { name: "Renamed Book" }], + ["GET", "/api/books/1", { data: null }], + ["GET", "/api/books/1", { data: RENAMED }], + ]; + // Serving still answers the renamed book, so the read that shows none is a check it fails, not one it skips. + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: false, + reason: "contradicted", + contradicted_by: 4, + }); + }); + + it("confirms no create by the absence of the entity a later delete removed", () => { + const rows: TimedCaptureRow[] = [ + ...BOOK_INIT_ROWS, + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + ["DELETE", "/api/books/3", "", 204], + ["GET", "/api/books/3", BOOK_NOT_FOUND, 404], + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + ]; + expect(gated(rows, "post_books")).toMatchObject({ + served: false, + reason: "unconfirmed", + checked_record_ordinals: [], + }); + expect(gated(rows, "delete_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [5, 6], + }); + }); + + it("confirms no create whose entity lacks a field every recording of its item view carried", () => { + const detailed = (entry: Book): Book => ({ ...entry, detail: "x" }); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", detailed(FIRST_BOOK)], + ["GET", "/api/books/2", detailed(SECOND_BOOK)], + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + ["GET", "/api/books/3", THIRD_BOOK], + ]; + expect(gated(rows, "post_books")).toMatchObject({ served: false }); + }); +}); + +describe("updates whose effect later writes erase", () => { + const versioned = (entry: Book, version: number): Book => ({ + ...entry, + version, + }); + + it("keeps an update confirmed when a later one rewrites only part of what it changed", () => { + const again = versioned(renamedBook(FIRST_BOOK, "Again Book", 4), 2); + const rows: TimedCaptureRow[] = [ + [ + "GET", + "/api/books", + [versioned(FIRST_BOOK, 1), versioned(SECOND_BOOK, 1)], + ], + ["GET", "/api/books/1", versioned(FIRST_BOOK, 1)], + ["GET", "/api/books/2", versioned(SECOND_BOOK, 1)], + [ + "PATCH", + "/api/books/1", + versioned(RENAMED, 2), + 200, + { name: "Renamed Book" }, + ], + ["PUT", "/api/books/1", again, 200, { name: "Again Book" }], + ["GET", "/api/books/1", again], + ]; + // The PUT keeps the version the PATCH incremented, so the read still shows the PATCH's effect. + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [5], + }); + }); + + it("confirms no update whose changes later writes overwrote between them", () => { + const renamed = versioned(renamedBook(FIRST_BOOK, "Again Book", 4), 2); + const final = versioned(renamedBook(FIRST_BOOK, "Again Book", 5), 9); + const rows: TimedCaptureRow[] = [ + [ + "GET", + "/api/books", + [versioned(FIRST_BOOK, 1), versioned(SECOND_BOOK, 1)], + ], + ["GET", "/api/books/1", versioned(FIRST_BOOK, 1)], + ["GET", "/api/books/2", versioned(SECOND_BOOK, 1)], + [ + "PATCH", + "/api/books/1", + versioned(RENAMED, 2), + 200, + { name: "Renamed Book" }, + ], + ["PUT", "/api/books/1", renamed, 200, { name: "Again Book" }], + ["POST", "/api/books/1", final, 200, { version: 9 }], + ["GET", "/api/books/1", final], + ]; + // The PUT overwrote the name, slug and clock, the POST the version: nothing of the PATCH is left to read. + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: false, + checked_record_ordinals: [], + }); + expect(gated(rows, "put_books_by_id")).toMatchObject({ + checked_record_ordinals: [6], + }); + }); + + it("confirms no update whose changes a recorded outcome replaced", () => { + const described = ( + entry: Book, + name: string, + at: number, + text: string, + ): Book => ({ + ...renamedBook(entry, name, at), + description: text, + }); + const final = described(FIRST_BOOK, "Other Book", 7, "z"); + const rows: TimedCaptureRow[] = [ + [ + "GET", + "/api/books", + [ + { ...FIRST_BOOK, description: "d" }, + { ...SECOND_BOOK, description: "d" }, + ], + ], + ["GET", "/api/books/1", { ...FIRST_BOOK, description: "d" }], + ["GET", "/api/books/2", { ...SECOND_BOOK, description: "d" }], + [ + "PUT", + "/api/books/2", + described(SECOND_BOOK, "Second Book", 3, "x"), + 200, + { description: "x" }, + ], + [ + "PATCH", + "/api/books/1", + described(FIRST_BOOK, "Renamed Book", 4, "d"), + 200, + { name: "Renamed Book" }, + ], + // Without a body the PUT is not modelled: its recorded outcome becomes the book's values. + ["PUT", "/api/books/1", described(FIRST_BOOK, "Other Book", 5, "d")], + [ + "PATCH", + "/api/books/1", + described(FIRST_BOOK, "Other Book", 6, "y"), + 200, + { description: "y" }, + ], + ["PUT", "/api/books/1", final, 200, { description: "z" }], + ["GET", "/api/books/1", final], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: false, + checked_record_ordinals: [], + }); + }); +}); + +describe("reads as serving answers them", () => { + it("checks a read with no query on the line serving answers it from, a whole list's paging line", () => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books?limit=50", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK, THIRD_BOOK]], + ]; + expect(gated(rows, "post_books")).toMatchObject({ + served: true, + checked_record_ordinals: [4], + }); + }); + + it("checks nothing by a line whose query repeats a name, which no call can name", () => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books?tag=a&tag=b", [FIRST_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + ["PATCH", "/api/books/1", RENAMED, 200, { name: "Renamed Book" }], + ["GET", "/api/books?tag=a&tag=b", [RENAMED]], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + checked_record_ordinals: [], + }); + }); + + it("checks nothing by a read made while a write to its targets was still running", () => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + // The create runs from second 3 to 4.5; the list at second 4 still shows the books before it. + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }, 1500], + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK, THIRD_BOOK]], + ]; + expect(gated(rows, "post_books")).toMatchObject({ + served: true, + checked_record_ordinals: [5], + contradicted_by: null, + }); + }); + + it("refuses an update a later call stores otherwise than the plan echoed", () => { + const formatted = (at: number, format: string): Book => ({ + ...renamedBook(FIRST_BOOK, "First Book", at), + format, + }); + const rows: TimedCaptureRow[] = [ + [ + "GET", + "/api/books", + [ + { ...FIRST_BOOK, format: "EBOOK" }, + { ...SECOND_BOOK, format: "EBOOK" }, + ], + ], + ["GET", "/api/books/1", { ...FIRST_BOOK, format: "EBOOK" }], + ["GET", "/api/books/2", { ...SECOND_BOOK, format: "EBOOK" }], + [ + "PATCH", + "/api/books/1", + formatted(3, "PAPER"), + 200, + { format: "PAPER" }, + ], + ["GET", "/api/books/1", formatted(3, "PAPER")], + [ + "PATCH", + "/api/books/1", + formatted(5, "HARDCOVER"), + 200, + { format: "hardcover" }, + ], + ["GET", "/api/books/1", formatted(5, "HARDCOVER")], + ]; + // One call echoed the format, the other stored another value: no one plan serves both. + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }); + + it("learns no update's effect from a read made while the update was still running", () => { + const full = (description: string) => ({ + id: 1, + name: "First", + description, + }); + const summary = [ + { id: 1, name: "First" }, + { id: 2, name: "Second" }, + ]; + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", summary], + ["GET", "/api/books/1", full("Old desc")], + ["GET", "/api/books/2", { id: 2, name: "Second", description: "Other" }], + // The update runs from second 3 to 4.5; the read at second 4 still shows the old description. + [ + "PATCH", + "/api/books/1", + "", + 204, + { name: "First", description: "New desc" }, + 1500, + ], + ["GET", "/api/books/1", full("Old desc")], + ["GET", "/api/books", summary], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: false, + observation: null, + }); + }); + + it("explains a field an update adds that the observing view had shown the entity without", () => { + const rows: TimedCaptureRow[] = [ + [ + "GET", + "/api/books", + [ + { id: 1, name: "A" }, + { id: 2, name: "B" }, + ], + ], + ["GET", "/api/books/1", { id: 1, name: "A" }], + ["GET", "/api/books/2", { id: 2, name: "B", edited_at: bookClock(0) }], + ["PATCH", "/api/books/1", "", 204, { name: "Z" }], + ["GET", "/api/books/1", { id: 1, name: "Z", edited_at: bookClock(3) }], + [ + "GET", + "/api/books", + [ + { id: 1, name: "Z" }, + { id: 2, name: "B" }, + ], + ], + ]; + const plan = gated(rows, "patch_books_by_id"); + expect(plan).toMatchObject({ served: true }); + expect(plan.rules).toContainEqual( + expect.objectContaining({ field: "edited_at", rule: "clock" }), + ); + }); +}); + +describe("what a read shows of a write", () => { + it("confirms no update by a read that shows none of the fields it changed", () => { + const rows: TimedCaptureRow[] = [ + [ + "GET", + "/api/books", + [ + { id: 1, name: "A" }, + { id: 2, name: "B" }, + ], + ], + ["GET", "/api/books/1", { id: 1, name: "A", description: "x" }], + ["GET", "/api/books/2", { id: 2, name: "B", description: "y" }], + [ + "PATCH", + "/api/books/1", + { id: 1, name: "A", description: "new" }, + 200, + { description: "new" }, + ], + // The list passes, but it shows no description. + [ + "GET", + "/api/books", + [ + { id: 1, name: "A" }, + { id: 2, name: "B" }, + ], + ], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: false, + reason: "unconfirmed", + checked_record_ordinals: [], + }); + }); + + it("lets a read a create was observed through still contradict another write", () => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + ["PATCH", "/api/books/1", RENAMED, 200, { name: "Renamed Book" }], + ["GET", "/api/books/1", RENAMED], + // The create answers with its id alone, so the next list shows its entity; that list shows book 1 unrenamed. + ["POST", "/api/books", { id: 3 }, 201, { name: "Third Book" }], + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK, THIRD_BOOK]], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: false, + contradicted_by: 6, + }); + }); + + it("confirms no document write a later replacement overwrote", () => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/settings", { theme: "light", lang: "en" }], + [ + "PATCH", + "/api/settings", + { theme: "dark", lang: "en" }, + 200, + { theme: "dark" }, + ], + [ + "PUT", + "/api/settings", + { theme: "blue", lang: "fr" }, + 200, + { theme: "blue", lang: "fr" }, + ], + ["GET", "/api/settings", { theme: "blue", lang: "fr" }], + ]; + expect(gated(rows, "patch_settings")).toMatchObject({ + served: false, + checked_record_ordinals: [], + }); + expect(gated(rows, "put_settings")).toMatchObject({ + served: true, + checked_record_ordinals: [3], + }); + }); +}); + +describe("document writes later writes overwrite", () => { + it("confirms no document replacement a later merge overwrote", () => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/settings", { theme: "light", lang: "en" }], + [ + "PUT", + "/api/settings", + { theme: "dark", lang: "fr" }, + 200, + { theme: "dark", lang: "fr" }, + ], + [ + "PATCH", + "/api/settings", + { theme: "blue", lang: "de" }, + 200, + { theme: "blue", lang: "de" }, + ], + ["GET", "/api/settings", { theme: "blue", lang: "de" }], + ]; + expect(gated(rows, "put_settings")).toMatchObject({ + served: false, + checked_record_ordinals: [], + }); + expect(gated(rows, "patch_settings")).toMatchObject({ + served: true, + checked_record_ordinals: [3], + }); + }); + + it("confirms no document merge a later replacement of its object overwrote", () => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/settings", { prefs: { lang: "en" }, theme: "light" }], + [ + "PATCH", + "/api/settings", + { prefs: { lang: "fr" }, theme: "light" }, + 200, + { prefs: { lang: "fr" } }, + ], + // The replacement holds `lang`, so it replaces all of `prefs`, which the merge wrote. + [ + "PUT", + "/api/settings", + { prefs: { lang: "de" }, theme: "light" }, + 200, + { lang: "de" }, + ], + ["GET", "/api/settings", { prefs: { lang: "de" }, theme: "light" }], + ]; + expect(gated(rows, "patch_settings")).toMatchObject({ + served: false, + checked_record_ordinals: [], + }); + expect(gated(rows, "put_settings")).toMatchObject({ + served: true, + checked_record_ordinals: [3], + }); + }); +}); + +describe("reads named as serving names them", () => { + const orgBook = (id: number, org: string, name: string): Book => ({ + id, + org, + name, + }); + it("contradicts an update by a read spelled plainly of the encoded collection serving answers it from", () => { + const alpha = orgBook(1, "acme1", "Alpha"); + const renamed = orgBook(1, "acme1", "Renamed"); + const beta = orgBook(2, "beta2", "Beta"); + const renamedBeta = orgBook(2, "beta2", "Renamed Beta"); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/orgs/%61cme1/books", [alpha]], + ["GET", "/api/orgs/beta2/books", [beta]], + ["GET", "/api/orgs/%61cme1/books/1", alpha], + ["GET", "/api/orgs/beta2/books/2", beta], + ["PATCH", "/api/orgs/%61cme1/books/1", renamed, 200, { name: "Renamed" }], + [ + "PATCH", + "/api/orgs/beta2/books/2", + renamedBeta, + 200, + { name: "Renamed Beta" }, + ], + // Serving answers this plain spelling with the renamed book, which the recording shows absent. + ["GET", "/api/orgs/acme1/books/1", null], + ["GET", "/api/orgs/%61cme1/books/1", renamed], + ["GET", "/api/orgs/beta2/books/2", renamedBeta], + ]; + const { writes } = environmentStateViewOf(timedCaptureEvidence(rows), [ + "sibling-variance", + ]); + expect(writes.find((plan) => plan.effect === "update")).toMatchObject({ + served: false, + contradicted_by: 6, + }); + }); + + it("confirms a delete by a plain absence of an entity held under the encoded collection", () => { + const alpha = orgBook(1, "acme1", "Alpha"); + const beta = orgBook(2, "beta2", "Beta"); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/orgs/%61cme1/books", [alpha]], + ["GET", "/api/orgs/beta2/books", [beta]], + ["GET", "/api/orgs/%61cme1/books/1", alpha], + ["GET", "/api/orgs/beta2/books/2", beta], + ["DELETE", "/api/orgs/acme1/books/1", "", 204], + ["GET", "/api/orgs/acme1/books/1", BOOK_NOT_FOUND, 404], + ["DELETE", "/api/orgs/beta2/books/2", "", 204], + ["GET", "/api/orgs/beta2/books/2", BOOK_NOT_FOUND, 404], + ]; + const { writes } = environmentStateViewOf(timedCaptureEvidence(rows), [ + "sibling-variance", + ]); + expect(writes.find((plan) => plan.effect === "delete")).toMatchObject({ + served: true, + checked_record_ordinals: expect.arrayContaining([5]), + }); + }); + + it("learns a plain-path update from a later read under the encoded collection", () => { + const alpha = orgBook(1, "acme1", "Alpha"); + const renamed = orgBook(1, "acme1", "Renamed"); + const beta = orgBook(2, "beta2", "Beta"); + const renamedBeta = orgBook(2, "beta2", "Renamed Beta"); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/orgs/%61cme1/books", [alpha]], + ["GET", "/api/orgs/beta2/books", [beta]], + ["GET", "/api/orgs/%61cme1/books/1", alpha], + ["GET", "/api/orgs/beta2/books/2", beta], + ["PATCH", "/api/orgs/acme1/books/1", "", 204, { name: "Renamed" }], + ["GET", "/api/orgs/%61cme1/books/1", renamed], + ["GET", "/api/orgs/%61cme1/books/1", renamed], + [ + "PATCH", + "/api/orgs/beta2/books/2", + renamedBeta, + 200, + { name: "Renamed Beta" }, + ], + ["GET", "/api/orgs/beta2/books/2", renamedBeta], + ]; + const { writes } = environmentStateViewOf(timedCaptureEvidence(rows), [ + "sibling-variance", + ]); + expect(writes.find((plan) => plan.effect === "update")).toMatchObject({ + served: true, + observation: { ordinal: 5 }, + checked_record_ordinals: expect.arrayContaining([6]), + }); + }); + + it("checks a line whose query repeats a name with one value, which a call still names", () => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books?tag=a&tag=a", [FIRST_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + ["PATCH", "/api/books/1", RENAMED, 200, { name: "Renamed Book" }], + // The list still shows the old name, though the item read after it shows the rename. + ["GET", "/api/books?tag=a&tag=a", [FIRST_BOOK]], + ["GET", "/api/books/1", RENAMED], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: false, + contradicted_by: 4, + }); + }); + + it("ties a create named plainly to the encoded collection serving creates it in", () => { + const linked = (id: number, org: string, name: string) => ({ + id, + name, + href: `/orgs/${org}/books/${String(id)}`, + }); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/orgs/%61cme1/books", [linked(1, "acme1", "Alpha")]], + ["GET", "/api/orgs/beta2/books", [linked(2, "beta2", "Beta")]], + ["GET", "/api/orgs/%61cme1/books/1", linked(1, "acme1", "Alpha")], + ["GET", "/api/orgs/beta2/books/2", linked(2, "beta2", "Beta")], + [ + "POST", + "/api/orgs/acme1/books", + linked(3, "acme1", "Gamma"), + 201, + { name: "Gamma" }, + ], + ["GET", "/api/orgs/acme1/books/3", linked(3, "acme1", "Gamma")], + [ + "POST", + "/api/orgs/beta2/books", + linked(4, "beta2", "Delta"), + 201, + { name: "Delta" }, + ], + ]; + const { writes } = environmentStateViewOf(timedCaptureEvidence(rows), [ + "sibling-variance", + ]); + expect(writes.find((plan) => plan.effect === "create")).toMatchObject({ + served: true, + collection: "/api/orgs/%61cme1/books", + }); + }); +}); + +describe("writes named as serving names them", () => { + const orgBook = ( + id: number, + org: string, + name: string, + archived = false, + ) => ({ + id, + org, + name, + archived, + }); + const ALPHA = orgBook(1, "acme1", "Alpha"); + const BETA = orgBook(2, "beta2", "Beta"); + const RENAMED_BETA = orgBook(2, "beta2", "Renamed Beta"); + const INIT_ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/orgs/%61cme1/books", [ALPHA]], + ["GET", "/api/orgs/beta2/books", [BETA]], + ["GET", "/api/orgs/%61cme1/books/1", ALPHA], + ["GET", "/api/orgs/beta2/books/2", BETA], + ]; + const RENAME_BETA: readonly TimedCaptureRow[] = [ + [ + "PATCH", + "/api/orgs/beta2/books/2", + RENAMED_BETA, + 200, + { name: "Renamed Beta" }, + ], + ["GET", "/api/orgs/beta2/books/2", RENAMED_BETA], + ]; + const CREATE_DELTA: readonly TimedCaptureRow[] = [ + [ + "POST", + "/api/orgs/beta2/books", + orgBook(4, "beta2", "Delta"), + 201, + { name: "Delta" }, + ], + ]; + const planned = (rows: readonly TimedCaptureRow[], effect: string) => + environmentStateViewOf(timedCaptureEvidence(rows), [ + "sibling-variance", + ]).writes.find((plan) => plan.effect === effect); + + it("contradicts a create named plainly by a list of the encoded collection it went to that omits it", () => { + const gamma = orgBook(3, "acme1", "Gamma"); + const rows: TimedCaptureRow[] = [ + ...INIT_ROWS, + ["POST", "/api/orgs/acme1/books", gamma, 201, { name: "Gamma" }], + ["GET", "/api/orgs/acme1/books/3", gamma], + // Serving lists the create in the encoded collection, which the recording shows without it. + ["GET", "/api/orgs/%61cme1/books", [ALPHA]], + ...CREATE_DELTA, + ]; + expect(planned(rows, "create")).toMatchObject({ + served: false, + contradicted_by: 6, + }); + }); + + it("leaves unknown the entity a plainly named action changed under the encoded collection", () => { + const archived = orgBook(1, "acme1", "Alpha", true); + const rows: TimedCaptureRow[] = [ + ...INIT_ROWS, + ["POST", "/api/orgs/acme1/books/1/archive", archived, 200], + // The rename's answer also shows the archive, which no rule of the rename explains. + [ + "PATCH", + "/api/orgs/acme1/books/1", + orgBook(1, "acme1", "Renamed", true), + 200, + { name: "Renamed" }, + ], + ...RENAME_BETA, + ]; + expect(planned(rows, "update")).toMatchObject({ + served: true, + reason: null, + }); + }); + + it("checks no list of the encoded collection after a plainly named write at the collection", () => { + const gamma = orgBook(3, "acme1", "Gamma"); + const rows: TimedCaptureRow[] = [ + ...INIT_ROWS, + ["DELETE", "/api/orgs/acme1/books", "", 204], + ["POST", "/api/orgs/acme1/books", gamma, 201, { name: "Gamma" }], + ["GET", "/api/orgs/acme1/books/3", gamma], + // The cleared collection now lists only the create, which the state alone cannot show. + ["GET", "/api/orgs/%61cme1/books", [gamma]], + ...CREATE_DELTA, + ]; + expect(planned(rows, "create")).toMatchObject({ + served: true, + contradicted_by: null, + }); + }); + + it("checks no read under the encoded collection after a plainly named write above it", () => { + const rows: TimedCaptureRow[] = [ + ...INIT_ROWS, + [ + "PATCH", + "/api/orgs/%61cme1/books/1", + orgBook(1, "acme1", "Renamed"), + 200, + { name: "Renamed" }, + ], + ["DELETE", "/api/orgs/acme1", "", 204], + // The organisation's books went with it, which the state alone cannot show. + ["GET", "/api/orgs/%61cme1/books/1", BOOK_NOT_FOUND, 404], + ...RENAME_BETA, + ]; + expect(planned(rows, "update")).toMatchObject({ + served: true, + contradicted_by: null, + }); + }); +}); + +describe("resolved encoded item observations", () => { + it("matches the resolved raw ID when the collection alias and response ID are encoded", () => { + const collection = "/api/orgs/%61cme1/books"; + const state = new SimulatedState( + [ + { + type: "/api/orgs/{org_id}/books", + collection, + id: "%61", + fields: { id: "%61", name: "Before" }, + view_fields: [], + record_ordinals: [], + }, + ], + [], + ); + const write = { + ordinal: 0, + method: "PATCH", + path: "/api/orgs/acme1/books/%61", + query: undefined, + status: 204, + start: 0, + end: 0, + }; + const target = itemTarget(state, "/api/orgs/{org_id}/books", write); + const itemView = { + tool: "get_books_by_id", + type: "/api/orgs/{org_id}/books", + shape: "item" as const, + pointer: "", + fields: ["id", "name"], + required_fields: ["id"], + checked_record_ordinals: [1], + envelope_ordinal: 1, + }; + const search: ObservationSearch = { + records: [ + write, + { + ordinal: 1, + method: "GET", + path: "/api/orgs/acme1/books/%61", + query: undefined, + status: 200, + start: 1, + end: 1, + }, + ], + from: 0, + viewOf: () => itemView, + withBody: (_ordinal, use) => use({ id: "%61", name: "After" }), + }; + + expect(target).toEqual({ collection, id: "%61" }); + expect( + laterObservation(search, { + type: itemView.type, + ...target, + idField: "id", + held: () => true, + isWrite: (record) => record.method !== "GET", + sameItem: (record) => { + const resolved = itemTarget(state, itemView.type, record); + return ( + resolved.collection === target.collection && + resolved.id === target.id + ); + }, + }), + ).toEqual({ + ordinal: 1, + pointer: "", + object: { id: "%61", name: "After" }, + id: "%61", + }); + }); +}); + +describe("what a write's own answer pins down", () => { + it("observes a create that answers with its id alone only through a read of that id", () => { + const fourth = book(4, "Fourth Book", "2026-10-01T11:00:00Z"); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + ["POST", "/api/books", { id: 3 }, 201, { name: "Third Book" }], + // A book the capture never listed is not the one the create made. + ["GET", "/api/books/4", fourth], + ["GET", "/api/books/4", fourth], + ["GET", "/api/books/3", THIRD_BOOK], + ["GET", "/api/books/3", THIRD_BOOK], + ]; + expect(gated(rows, "post_books")).toMatchObject({ + observation: { ordinal: 6, pointer: "" }, + checked_record_ordinals: [7], + }); + }); + + it("keeps an update served when a later call is the first to send the path's id in its body", () => { + const renamedSecond = renamedBook(SECOND_BOOK, "Renamed Second", 5); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + ["PATCH", "/api/books/1", RENAMED, 200, { name: "Renamed Book" }], + ["GET", "/api/books/1", RENAMED], + [ + "PATCH", + "/api/books/2", + renamedSecond, + 200, + { id: 2, name: "Renamed Second" }, + ], + ["GET", "/api/books/2", renamedSecond], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + body: { identity: "id" }, + }); + }); +}); + +it("counts the sent values a plan keeps for fixed members against the view's allowance", () => { + let retained = 0; + const context = { + io: { retain: (bytes: number) => (retained += bytes) }, + } as unknown as EffectContext; + retainFixed(context, [{ sent: "x".repeat(10_000) }]); + expect(retained).toBeGreaterThan(20_000); + retained = 0; + retainFixed(context, []); + expect(retained).toBe(0); +}); + +describe("what a check leaves pending", () => { + const summary = (books: readonly Book[]): JsonValue => + books.map(({ id, name }) => ({ id: id ?? null, name: name ?? null })); + const described = (id: number, name: string, description: string) => ({ + id, + name, + description, + }); + + it("keeps an update a passing read showed nothing of for a later read to confirm", () => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", summary([FIRST_BOOK, SECOND_BOOK])], + ["GET", "/api/books/1", described(1, "First Book", "x")], + ["GET", "/api/books/2", described(2, "Second Book", "y")], + [ + "PATCH", + "/api/books/1", + described(1, "First Book", "new"), + 200, + { description: "new" }, + ], + // The list passes but shows no description; the item read does. + ["GET", "/api/books", summary([FIRST_BOOK, SECOND_BOOK])], + ["GET", "/api/books/1", described(1, "First Book", "new")], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [5], + }); + }); + + it("checks only a partial update's fields that remain after a later overwrite", () => { + const patched = described(1, "Patched", "survives"); + const replaced = described(1, "Replaced", "survives"); + const changedByOtherAction = described(1, "Archived", "survives"); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", summary([FIRST_BOOK, SECOND_BOOK])], + ["GET", "/api/books/1", described(1, "First Book", "original")], + ["GET", "/api/books/2", described(2, "Second Book", "other")], + [ + "PATCH", + "/api/books/1", + patched, + 200, + { name: "Patched", description: "survives" }, + ], + ["PUT", "/api/books/1", replaced, 200, { name: "Replaced" }], + ["POST", "/api/books/1/archive", changedByOtherAction, 204, {}], + ["GET", "/api/books/1", changedByOtherAction], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [6], + contradicted_by: null, + }); + }); + + it("checks a surviving sent member after a derived field was bumped by an unmodelled write", () => { + const entry = ( + name: string, + description: string, + version: number, + ): Book => ({ id: 1, name, description, version }); + const other = { id: 2, name: "B", description: "y", version: 1 }; + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [entry("A", "x", 1), other]], + ["GET", "/api/books/1", entry("A", "x", 1)], + ["GET", "/api/books/2", other], + [ + "PATCH", + "/api/books/1", + entry("A2", "new", 2), + 200, + { name: "A2", description: "new" }, + ], + ["PUT", "/api/books/1", entry("A3", "new", 2), 200, { name: "A3" }], + ["POST", "/api/books/1/archive", { archived: true }, 204, {}], + // The read still shows the description the patch sent. The version bump is not that patch. + ["GET", "/api/books/1", entry("A3", "new", 9)], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + rules: expect.arrayContaining([{ field: "version", rule: "increment" }]), + checked_record_ordinals: [6], + contradicted_by: null, + }); + }); + + it("neither resynchronises nor checks an entity by a read begun while a write to it still ran", () => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + // The rename runs from second 3 to 5.5, and an unmodelled action leaves book 1 unknown. + ["PATCH", "/api/books/1", RENAMED, 200, { name: "Renamed Book" }, 2500], + ["POST", "/api/books/1/archive", null, 204, {}], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/1", RENAMED], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [6], + contradicted_by: null, + }); + }); + + it("learns an update from a read begun after the write, not from one that overlapped it", () => { + const named = (name: string) => ({ id: 1, name }); + const other = { id: 2, name: "B" }; + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [named("A"), other]], + ["GET", "/api/books/1", named("A")], + ["GET", "/api/books/2", other], + // The patch runs from second 3 to 5.5 and returns no entity. A read begun while it still runs shows the old name. + ["PATCH", "/api/books/1", "", 204, { name: "Z" }, 2500], + ["GET", "/api/books/1", named("A")], + ["GET", "/api/books/1", named("A")], + ["GET", "/api/books/1", named("Z")], + ["GET", "/api/books", [named("Z"), other]], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + observation: { ordinal: 6, pointer: "" }, + body: { accepted: ["name"], fixed: [] }, + rules: [{ field: "name", rule: "echo" }], + checked_record_ordinals: [7], + contradicted_by: null, + }); + }); + + it("checks nothing by a range read the collection has outgrown", () => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books?_start=0&_end=2", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + // The application still answers the range with two books. + ["GET", "/api/books?_start=0&_end=2", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/3", THIRD_BOOK], + ]; + expect(gated(rows, "post_books")).toMatchObject({ + served: true, + checked_record_ordinals: [5], + }); + }); +}); + +describe("what a later call must follow", () => { + it("rejects a later create whose unique field repeats a value already held", () => { + const existing = { + id: "123e4567-e89b-42d3-a456-426614174000", + ref: "223e4567-e89b-42d3-a456-426614174000", + name: "Existing", + }; + const first = { + id: "323e4567-e89b-42d3-a456-426614174000", + ref: "423e4567-e89b-42d3-a456-426614174000", + name: "First", + }; + const duplicate = { + id: "523e4567-e89b-42d3-a456-426614174000", + ref: first.ref, + name: "Second", + }; + const rows: TimedCaptureRow[] = [ + ["GET", "/api/items", [existing]], + ["GET", `/api/items/${existing.id}`, existing], + ["POST", "/api/items", first, 201, { name: "First" }], + ["GET", `/api/items/${first.id}`, first], + ["GET", "/api/items", [existing, first]], + ["POST", "/api/items", duplicate, 201, { name: "Second" }], + ["GET", `/api/items/${duplicate.id}`, duplicate], + ["GET", "/api/items", [existing, first, duplicate]], + ]; + expect(gated(rows, "post_items")).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }); + + it("refuses a create whose later call does not follow the sequence it learned", () => { + const ranked = (at: Book, rank: number) => ({ ...at, rank }); + const third = ranked(book(3, "Third Book", bookClock(3)), 3); + const fourth = ranked(book(4, "Fourth Book", bookClock(4)), 3); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [ranked(FIRST_BOOK, 1), ranked(SECOND_BOOK, 2)]], + ["GET", "/api/books/1", ranked(FIRST_BOOK, 1)], + ["GET", "/api/books/2", ranked(SECOND_BOOK, 2)], + ["POST", "/api/books", third, 201, { name: "Third Book" }], + // Serving ranks the next create 4; the application recorded 3 again. + ["POST", "/api/books", fourth, 201, { name: "Fourth Book" }], + [ + "GET", + "/api/books", + [ranked(FIRST_BOOK, 1), ranked(SECOND_BOOK, 2), third, fourth], + ], + ]; + expect(gated(rows, "post_books")).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }); + + it("keeps a document write a later replacement of another member left in place", () => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/settings", { theme: "light", prefs: { lang: "en" } }], + [ + "PATCH", + "/api/settings", + { theme: "dark", prefs: { lang: "en" } }, + 200, + { theme: "dark" }, + ], + // The replacement holds `lang`, so it replaces `prefs` alone. + [ + "PUT", + "/api/settings", + { theme: "dark", prefs: { lang: "fr" } }, + 200, + { lang: "fr" }, + ], + ["GET", "/api/settings", { theme: "dark", prefs: { lang: "fr" } }], + ]; + expect(gated(rows, "patch_settings")).toMatchObject({ + served: true, + checked_record_ordinals: [3], + }); + expect(gated(rows, "put_settings")).toMatchObject({ + served: true, + checked_record_ordinals: [3], + }); + }); + + it("applies a later document replacement at the plan's pointer, though it shares no member with the document", () => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/settings", { theme: "light", lang: "en" }], + [ + "PUT", + "/api/settings", + { theme: "dark", lang: "fr" }, + 200, + { theme: "dark", lang: "fr" }, + ], + ["PUT", "/api/settings", { theme: "blue" }, 200, { theme: "blue" }], + ["PUT", "/api/settings", { lang: "de" }, 200, { lang: "de" }], + ["GET", "/api/settings", { lang: "de" }], + ]; + expect(gated(rows, "put_settings")).toMatchObject({ + served: true, + checked_record_ordinals: [4], + }); + }); + + it("finds an update's entity as serving does when the write spells the collection plainly", () => { + const alpha = { id: 1, org: "acme1", name: "Alpha" }; + const beta = { id: 2, org: "beta2", name: "Beta" }; + const renamed = { ...alpha, name: "Renamed" }; + const renamedBeta = { ...beta, name: "Renamed Beta" }; + const rows: TimedCaptureRow[] = [ + ["GET", "/api/orgs/%61cme1/books", [alpha]], + ["GET", "/api/orgs/beta2/books", [beta]], + ["GET", "/api/orgs/%61cme1/books/1", alpha], + ["GET", "/api/orgs/beta2/books/2", beta], + ["PATCH", "/api/orgs/acme1/books/1", renamed, 200, { name: "Renamed" }], + [ + "PATCH", + "/api/orgs/beta2/books/2", + renamedBeta, + 200, + { name: "Renamed Beta" }, + ], + ["GET", "/api/orgs/%61cme1/books/1", renamed], + ["GET", "/api/orgs/beta2/books/2", renamedBeta], + ]; + const { writes } = environmentStateViewOf(timedCaptureEvidence(rows), [ + "sibling-variance", + ]); + expect(writes.find((plan) => plan.effect === "update")).toMatchObject({ + served: true, + checked_record_ordinals: [6, 7], + }); + }); +}); + +describe("what a write's effect keeps", () => { + it("keeps an entity member a later write's other member does not cover, though both names look like pointers", () => { + const flat = (id: number, prefs: string, lang: string) => ({ + id, + name: `Book ${String(id)}`, + "/prefs": prefs, + "/prefs/lang": lang, + }); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [flat(1, "x", "en"), flat(2, "x", "en")]], + ["GET", "/api/books/1", flat(1, "x", "en")], + ["GET", "/api/books/2", flat(2, "x", "en")], + [ + "PATCH", + "/api/books/1", + flat(1, "x", "fr"), + 200, + { "/prefs/lang": "fr" }, + ], + ["PUT", "/api/books/1", flat(1, "y", "fr"), 200, { "/prefs": "y" }], + ["GET", "/api/books/1", flat(1, "y", "fr")], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [5], + }); + }); + + const subtitled = (id: number, name: string, subtitle?: string) => + subtitle === undefined ? { id, name } : { id, name, subtitle }; + const summaries = (first: string): TimedCaptureRow => [ + "GET", + "/api/books", + [subtitled(1, first), subtitled(2, "B")], + ]; + + it("refuses an update a later read shows dropped a field serving would keep", () => { + const rows: TimedCaptureRow[] = [ + summaries("A"), + ["GET", "/api/books/1", subtitled(1, "A", "s")], + ["GET", "/api/books/2", subtitled(2, "B", "t")], + ["PATCH", "/api/books/1", "", 204, { name: "Z" }], + ["GET", "/api/books/1", subtitled(1, "Z")], + summaries("Z"), + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }); + + it("keeps an update whose partial response leaves out a field a later read still shows", () => { + const rows: TimedCaptureRow[] = [ + summaries("A"), + ["GET", "/api/books/1", subtitled(1, "A", "s")], + ["GET", "/api/books/2", subtitled(2, "B", "t")], + ["PATCH", "/api/books/1", subtitled(1, "Z"), 200, { name: "Z" }], + ["GET", "/api/books/1", subtitled(1, "Z", "s")], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [4], + }); + }); +}); + +describe("what an unmodelled write explains", () => { + it("weighs an unmodelled write against a tool's call on the entity a read checks, not its latest call elsewhere", () => { + const tagged = (id: number, name: string, tag: string) => ({ + id, + name, + tag, + }); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [tagged(1, "A", "x"), tagged(2, "B", "x")]], + ["GET", "/api/books/1", tagged(1, "A", "x")], + ["GET", "/api/books/2", tagged(2, "B", "x")], + ["PATCH", "/api/books/1", tagged(1, "A2", "x"), 200, { name: "A2" }], + ["POST", "/api/exports", { queued: true }, 202, { format: "csv" }], + ["PATCH", "/api/books/2", tagged(2, "B2", "x"), 200, { name: "B2" }], + // The export came after book 1's rename, so it may explain the changed tag; the rename itself shows. + ["GET", "/api/books/1", tagged(1, "A2", "y")], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + contradicted_by: null, + checked_record_ordinals: [6], + }); + }); + + it("lets an unmodelled write explain a list's change to an entity a tool made before it, whatever the tool did after", () => { + const fourth = book(4, "Fourth Book", bookClock(5)); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + ["POST", "/api/exports", { queued: true }, 202, { format: "csv" }], + ["POST", "/api/books", fourth, 201, { name: "Fourth Book" }], + ["GET", "/api/books/4", fourth], + // The export came after the third book was made, so it may explain the third book's other name. + [ + "GET", + "/api/books", + [FIRST_BOOK, SECOND_BOOK, { ...THIRD_BOOK, name: "Retitled" }, fourth], + ], + ]; + expect(gated(rows, "post_books")).toMatchObject({ + served: true, + contradicted_by: null, + }); + }); + + it("weighs no unmodelled write against a call whose effect a later write overwrote", () => { + const named = (id: number, name: string) => ({ + ...book(id, name, bookClock(0)), + id, + }); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [named(1, "A"), named(2, "B")]], + ["GET", "/api/books/1", named(1, "A")], + ["GET", "/api/books/2", named(2, "B")], + ["PATCH", "/api/books/1", named(1, "A2"), 200, { name: "A2" }], + ["POST", "/api/exports", { queued: true }, 202, { format: "csv" }], + // The PUT overwrites the patch's only change on book 1. + ["PUT", "/api/books/1", named(1, "A3"), 200, { name: "A3" }], + ["PATCH", "/api/books/2", named(2, "B2"), 200, { name: "B2" }], + ["GET", "/api/books/2", named(2, "B2")], + // The export came before book 2's patch, which the list shows undone. + ["GET", "/api/books", [named(1, "A3"), named(2, "B")]], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: false, + contradicted_by: 8, + }); + }); + + it("contradicts by a failed item read the writes no unmodelled write came after, checking only the others' members", () => { + const enveloped = (data: Record, revision: string) => ({ + data, + revision, + }); + const full = (name: string, description: string) => ({ + id: 1, + name, + description, + }); + const summary = [ + { id: 1, name: "A" }, + { id: 2, name: "B" }, + ]; + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", summary], + ["GET", "/api/books/1", enveloped(full("A", "x"), "r1")], + [ + "GET", + "/api/books/2", + enveloped({ id: 2, name: "B", description: "y" }, "r1"), + ], + ["PATCH", "/api/books/1", full("A", "new"), 200, { description: "new" }], + ["GET", "/api/books", summary], + ["POST", "/api/exports", { queued: true }, 202, { format: "csv" }], + // The rename sends the name alone, so the earlier description patch stays pending. + ["PUT", "/api/books/1", full("Z", "new"), 200, { name: "Z" }], + // The rename shows, but the envelope serving would answer with does not. + ["GET", "/api/books/1", enveloped(full("Z", "new"), "r2")], + ]; + expect(gated(rows, "put_books_by_id")).toMatchObject({ + served: false, + contradicted_by: 7, + }); + }); + + it("contradicts by a list a write made after an unmodelled one, though the tool's earlier change elsewhere came before it", () => { + const named = (id: number, name: string) => ({ + ...book(id, name, bookClock(0)), + id, + }); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [named(1, "A"), named(2, "B")]], + ["GET", "/api/books/1", named(1, "A")], + ["GET", "/api/books/2", named(2, "B")], + ["PATCH", "/api/books/1", named(1, "A2"), 200, { name: "A2" }], + ["POST", "/api/exports", { queued: true }, 202, { format: "csv" }], + ["PATCH", "/api/books/2", named(2, "B2"), 200, { name: "B2" }], + ["GET", "/api/books/2", named(2, "B2")], + // Only book 2 differs, and its patch came after the export. + ["GET", "/api/books", [named(1, "A2"), named(2, "B")]], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: false, + contradicted_by: 7, + }); + }); + + it("contradicts a write made after an unmodelled one, though an earlier pending write came before it", () => { + const summary = (name: string) => [ + { id: 1, name }, + { id: 2, name: "B" }, + ]; + const full = (name: string, description: string) => ({ + id: 1, + name, + description, + }); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", summary("A")], + ["GET", "/api/books/1", full("A", "x")], + ["GET", "/api/books/2", { id: 2, name: "B", description: "y" }], + ["PATCH", "/api/books/1", full("A", "new"), 200, { description: "new" }], + ["GET", "/api/books", summary("A")], + ["POST", "/api/exports", { queued: true }, 202, { format: "csv" }], + [ + "PUT", + "/api/books/1", + full("Z", "new"), + 200, + { name: "Z", description: "new" }, + ], + ["GET", "/api/books/1", full("Z", "new")], + // The export came before the rename, so it cannot explain the list showing the old name. + ["GET", "/api/books", summary("A")], + ]; + expect(gated(rows, "put_books_by_id")).toMatchObject({ + served: false, + contradicted_by: 8, + }); + }); +}); + +describe("reads and calls on an entity an unmodelled write left unknown", () => { + const named = (name: string) => ({ id: 1, name }); + const other = { id: 2, name: "B" }; + const ARCHIVE: TimedCaptureRow = [ + "POST", + "/api/books/1/archive", + null, + 204, + {}, + ]; + + it("keeps the entity unknown through a failed read that is no absence, for a later read to check", () => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [named("A"), other]], + ["GET", "/api/books/1", named("A")], + ["GET", "/api/books/2", other], + ["PATCH", "/api/books/1", named("Z"), 200, { name: "Z" }], + ARCHIVE, + // The application was briefly unavailable, so this answer shows nothing of the book. + ["GET", "/api/books/1", { error: "unavailable" }, 503], + ["GET", "/api/books/1", named("Z")], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [6], + contradicted_by: null, + }); + }); + + it("confirms no update of a volatile field by a read that shows the entity again, as no other read does", () => { + const seen = (id: number, count: number) => ({ + id, + name: id === 1 ? "A" : "B", + seen: count, + }); + const rows = (archive: boolean): TimedCaptureRow[] => [ + ["GET", "/api/books", [seen(1, 1), seen(2, 1)]], + ["GET", "/api/books/1", seen(1, 1)], + // The same read again shows another count, so the count changes with no write. + ["GET", "/api/books/1", seen(1, 2)], + ["GET", "/api/books/2", seen(2, 1)], + ["PATCH", "/api/books/1", seen(1, 5), 200, { seen: 5 }], + ...(archive ? [ARCHIVE] : []), + ["GET", "/api/books/1", seen(1, 9)], + ]; + for (const archive of [false, true]) + expect(gated(rows(archive), "patch_books_by_id")).toMatchObject({ + served: false, + reason: "unconfirmed", + checked_record_ordinals: [], + }); + }); + + it("adds nothing to an update's plan from a call it cannot check", () => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [named("A"), other]], + ["GET", "/api/books/1", named("A")], + ["GET", "/api/books/2", other], + ["PATCH", "/api/books/1", named("Z"), 200, { name: "Z" }], + ["GET", "/api/books/1", named("Z")], + ARCHIVE, + // The application stored this name otherwise than sent; a call on a known book would conflict with the echo. + ["PATCH", "/api/books/1", named("Y!"), 200, { name: "y" }], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + body: { accepted: ["name"], fixed: [] }, + rules: [{ field: "name", rule: "echo" }], + checked_record_ordinals: [4], + }); + }); +}); + +describe("volatile fields", () => { + it("confirms no write through a volatile field a read renders from itself", () => { + const seen = (id: number, name: string, at: string) => ({ + id, + name, + seen: at, + }); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [seen(1, "A", "t1"), seen(2, "B", "t1")]], + ["GET", "/api/books/1", seen(1, "A", "t1")], + // A repeated read with no write between shows `seen` changing on its own, so it is volatile. + ["GET", "/api/books/1", seen(1, "A", "t2")], + ["GET", "/api/books/2", seen(2, "B", "t1")], + ["PATCH", "/api/books/1", seen(1, "A", "t3"), 200, { seen: "t3" }], + ["POST", "/api/exports", { queued: true }, 202, { format: "csv" }], + // The export may explain the new name, and `seen` renders from the read, so nothing shows the patch. + ["GET", "/api/books/1", seen(1, "Z", "t9")], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: false, + checked_record_ordinals: [], + }); + }); +}); + +describe("an unmodelled write under another collection spelling", () => { + it("keeps a confirmed update when the write spells the held collection plainly", () => { + const acme = (name: string, status: string) => ({ + id: 1, + org: "acme1", + name, + status, + }); + const beta = (name: string) => ({ + id: 2, + org: "beta2", + name, + status: "open", + }); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/orgs/%61cme1/books", [acme("Alpha", "open")]], + ["GET", "/api/orgs/beta2/books", [beta("Beta")]], + ["GET", "/api/orgs/%61cme1/books/1", acme("Alpha", "open")], + ["GET", "/api/orgs/beta2/books/2", beta("Beta")], + [ + "PATCH", + "/api/orgs/acme1/books/1", + acme("Renamed", "open"), + 200, + { name: "Renamed" }, + ], + [ + "PATCH", + "/api/orgs/beta2/books/2", + beta("Renamed Beta"), + 200, + { name: "Renamed Beta" }, + ], + ["GET", "/api/orgs/%61cme1/books/1", acme("Renamed", "open")], + ["GET", "/api/orgs/beta2/books/2", beta("Renamed Beta")], + // Book 1 is held under the encoded collection. This archive spells it plainly. + ["POST", "/api/orgs/acme1/books/1/archive", null, 204, {}], + // The response shows the status the archive changed. The patch sent only the name. + [ + "PATCH", + "/api/orgs/%61cme1/books/1", + acme("Retitled", "archived"), + 200, + { name: "Retitled" }, + ], + ]; + const { writes } = environmentStateViewOf(timedCaptureEvidence(rows), [ + "sibling-variance", + ]); + expect(writes.find((plan) => plan.effect === "update")).toMatchObject({ + served: true, + reason: null, + checked_record_ordinals: [6, 7], + }); + }); +}); + +describe("timed write observations and generated clocks", () => { + it("does not learn an update from a read that began before the write completed", () => { + const changed = { ...FIRST_BOOK, name: "Changed", slug: "changed" }; + const rows: TimedCaptureRow[] = [ + ...BOOK_INIT_ROWS, + [ + "PATCH", + "/api/books/1", + { name: "Changed" }, + 200, + { name: "Changed" }, + 2_000, + ], + // This read starts before the PATCH ends and still shows the old value. + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/1", changed], + ["GET", "/api/books/1", changed], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + observation: { ordinal: 5 }, + checked_record_ordinals: [6], + }); + }); + + it("refuses a later update that leaves a learned clock unchanged", () => { + const afterFirst = { + ...FIRST_BOOK, + name: "First update", + updated_at: bookClock(4), + }; + const afterSecond = { ...afterFirst, name: "Second update" }; + const rows: TimedCaptureRow[] = [ + ...BOOK_INIT_ROWS, + ["PATCH", "/api/books/1", afterFirst, 200, { name: "First update" }], + ["GET", "/api/books/1", afterFirst], + [ + "PATCH", + "/api/books/1", + { id: 1, name: "Second update" }, + 200, + { name: "Second update" }, + ], + ["GET", "/api/books/1", afterSecond], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }); + + it("keeps a later update whose regenerated clock advances", () => { + const afterFirst = { + ...FIRST_BOOK, + name: "First update", + updated_at: bookClock(4), + }; + const afterSecond = { + ...afterFirst, + name: "Second update", + updated_at: bookClock(6), + }; + const rows: TimedCaptureRow[] = [ + ...BOOK_INIT_ROWS, + [ + "PATCH", + "/api/books/1", + { name: "First update" }, + 200, + { name: "First update" }, + ], + ["GET", "/api/books/1", afterFirst], + [ + "PATCH", + "/api/books/1", + { id: 1, name: "Second update" }, + 200, + { name: "Second update" }, + ], + ["GET", "/api/books/1", afterSecond], + ["GET", "/api/books/1", afterSecond], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [7], + }); + }); + + it("leaves a later update unknown when neither its answer nor a read before the next write shows its learned clock", () => { + const summary = ({ id, name }: Book): Book => ({ id: id!, name: name! }); + const renamed = renamedBook(FIRST_BOOK, "Renamed Book", 3); + const retitled = renamedBook(renamed, "Retitled Book", 5); + const rows = (answer: JsonValue, status: number): TimedCaptureRow[] => [ + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", summary(FIRST_BOOK)], + ["GET", "/api/books/2", summary(SECOND_BOOK)], + ["PATCH", "/api/books/1", renamed, 200, { name: "Renamed Book" }], + ["GET", "/api/books", [renamed, SECOND_BOOK]], + ["PATCH", "/api/books/1", answer, status, { name: "Retitled Book" }], + // The item view shows no clock; the list shows the one the second call stamped. + ["GET", "/api/books/1", summary(retitled)], + ["GET", "/api/books", [retitled, SECOND_BOOK]], + ]; + expect(gated(rows("", 204), "patch_books_by_id")).toMatchObject({ + served: true, + rules: expect.arrayContaining([ + expect.objectContaining({ field: "updated_at", rule: "clock" }), + ]), + checked_record_ordinals: [4], + contradicted_by: null, + }); + // An answer that shows the clock observes the call, so the list checks it. + expect(gated(rows(retitled, 200), "patch_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [4, 6, 7], + }); + }); +}); + +describe("partial update observations", () => { + it("keeps write-response fields when the item view omits them", () => { + const first: Book = { ...FIRST_BOOK, description: "First description" }; + const second: Book = { ...SECOND_BOOK, description: "Second description" }; + const itemSummary = (entry: Book): Book => ({ + id: entry.id!, + name: entry.name!, + }); + const renamed = { + ...first, + name: "Renamed first", + slug: "renamed-first", + updated_at: bookClock(3), + }; + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [first, second]], + ["GET", "/api/books/1", itemSummary(first)], + ["GET", "/api/books/2", itemSummary(second)], + ["PATCH", "/api/books/1", renamed, 200, { name: "Renamed first" }], + ["GET", "/api/books/1", itemSummary(renamed)], + [ + "PATCH", + "/api/books/2", + { id: 2, name: second.name!, description: "Revised description" }, + 200, + { description: "Revised description" }, + ], + // This item view covers only id/name, not description or the learned clock. + ["GET", "/api/books/2", itemSummary(second)], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + reason: null, + checked_record_ordinals: [4], + rules: [ + { field: "name", rule: "echo" }, + { field: "slug", rule: "slug", from: "name" }, + { field: "updated_at", rule: "clock" }, + { field: "description", rule: "echo" }, + ], + }); + }); + + it("uses read coverage without dropping fields shown by the write response", () => { + const first: Book = { ...FIRST_BOOK, description: "First description" }; + const second: Book = { ...SECOND_BOOK, description: "Second description" }; + const itemSummary = (entry: Book): Book => ({ + id: entry.id!, + name: entry.name!, + updated_at: entry.updated_at!, + }); + const renamed = { + ...first, + name: "Renamed first", + slug: "renamed-first", + updated_at: bookClock(3), + }; + const revised = { + id: 2, + name: second.name!, + description: "Revised description", + }; + const revisedSummary = { + id: 2, + name: second.name!, + updated_at: bookClock(6), + }; + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [first, second]], + ["GET", "/api/books/1", itemSummary(first)], + ["GET", "/api/books/2", itemSummary(second)], + ["PATCH", "/api/books/1", renamed, 200, { name: "Renamed first" }], + ["GET", "/api/books/1", itemSummary(renamed)], + [ + "PATCH", + "/api/books/2", + revised, + 200, + { description: "Revised description" }, + ], + // The item view covers the clock, but not the description returned by PATCH. + ["GET", "/api/books/2", revisedSummary], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + reason: null, + checked_record_ordinals: [4], + rules: [ + { field: "name", rule: "echo" }, + { field: "slug", rule: "slug", from: "name" }, + { field: "updated_at", rule: "clock" }, + { field: "description", rule: "echo" }, + ], + }); + }); +}); + +describe("what later calls must keep consistent", () => { + it("refuses an update one call sends an ignored id with and another the path's id", () => { + const renamed = (at: Book, name: string) => ({ ...at, name }); + const rows: TimedCaptureRow[] = [ + ...BOOK_INIT_ROWS, + [ + "PATCH", + "/api/books/1", + renamed(FIRST_BOOK, "B"), + 200, + { id: 999, name: "B" }, + ], + ["GET", "/api/books/1", renamed(FIRST_BOOK, "B")], + [ + "PATCH", + "/api/books/2", + renamed(SECOND_BOOK, "C"), + 200, + { id: 2, name: "C" }, + ], + ["GET", "/api/books/2", renamed(SECOND_BOOK, "C")], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }); + + it("refuses a create whose later call records a clock in another format", () => { + const epoch = (iso: string) => Math.floor(Date.parse(iso) / 1000); + const fourth = { + ...book(4, "Fourth Book", bookClock(5)), + created_at: epoch(bookClock(5)), + updated_at: epoch(bookClock(5)), + }; + const rows: TimedCaptureRow[] = [ + ...BOOK_INIT_ROWS, + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + ["GET", "/api/books/3", THIRD_BOOK], + ["POST", "/api/books", fourth, 201, { name: "Fourth Book" }], + ["GET", "/api/books/4", fourth], + ]; + expect(gated(rows, "post_books")).toMatchObject({ + served: false, + reason: "unexplained-change", + }); + }); + + it("names no entity by a delete whose path id decodes to it only twice, as serving would not", () => { + const uid = (key: string) => ({ uid: key, name: key }); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [uid("book1"), uid("book2")]], + ["GET", "/api/books/uid/book1", uid("book1")], + ["GET", "/api/books/uid/book2", uid("book2")], + ["DELETE", "/api/books/uid/book%2531", "", 204], + ["GET", "/api/books/uid/book1", BOOK_NOT_FOUND, 404], + ]; + const { writes } = environmentStateViewOf(timedCaptureEvidence(rows), [ + "after-uid-literal", + ]); + expect(writes.find((plan) => plan.effect === "delete")).toMatchObject({ + served: false, + checked_record_ordinals: [], + }); + }); + + it("checks nothing by a whole line recorded empty under a zero limit once a create fills its collection", () => { + const alpha = { id: 1, name: "Alpha", org: "1" }; + const beta = { id: 2, name: "Beta", org: "2" }; + const rows: TimedCaptureRow[] = [ + ["GET", "/api/orgs/1/books", [alpha]], + ["GET", "/api/orgs/1/books/1", alpha], + ["GET", "/api/orgs/2/books?limit=0", []], + ["POST", "/api/orgs/2/books", beta, 201, { name: "Beta" }], + ["GET", "/api/orgs/2/books/2", beta], + // An empty recording does not show that zero lists every entity. + ["GET", "/api/orgs/2/books?limit=0", []], + ]; + const { writes } = environmentStateViewOf(timedCaptureEvidence(rows)); + expect(writes.find((plan) => plan.effect === "create")).toMatchObject({ + served: true, + checked_record_ordinals: [4], + }); + }); +}); + +describe("path fields of a create", () => { + const held = "/api/orgs/uid/%2561cme1/books"; + /** A create held under a doubly encoded parent, then one whose call spells the parent once encoded, read back with `org`. */ + const rowsWith = (org: string): TimedCaptureRow[] => { + const alpha = { + ...book(1, "Alpha", "2026-10-01T09:00:00Z"), + org: "%61cme1", + }; + const made = { ...book(2, "Made", bookClock(2)), org: "%61cme1" }; + const other = { ...book(3, "Other", bookClock(3)), org }; + return [ + ["GET", held, [alpha]], + ["GET", `${held}/1`, alpha], + ["POST", held, made, 201, { name: "Made" }], + ["POST", "/api/orgs/uid/%61cme1/books", other, 201, { name: "Other" }], + ["GET", `${held}/3`, other], + ]; + }; + const planOf = (rows: TimedCaptureRow[]) => + environmentStateViewOf(timedCaptureEvidence(rows), [ + "after-uid-literal", + ]).writes.find((candidate) => candidate.effect === "create"); + + it("contradicts a later create whose read shows a parent serving would not write", () => { + expect(planOf(rowsWith("acme1"))).toMatchObject({ + served: false, + checked_record_ordinals: [], + contradicted_by: 4, + }); + }); + + it("confirms a later create whose read shows the parent decoded from the collection serving selects", () => { + expect(planOf(rowsWith("%61cme1"))).toMatchObject({ + served: true, + checked_record_ordinals: [4], + }); + }); +}); diff --git a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts new file mode 100644 index 0000000000..098e849a29 --- /dev/null +++ b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts @@ -0,0 +1,555 @@ +import { + at, + changesState, + isObject, + isStateWrite, + normalizedPath, + type JsonObject, + type StateEntity, + type StateRecordFacts, + type StateToolFacts, + type StateType, + type StateView, +} from "./environmentStateRules.js"; +import { updatedEntity } from "./environmentStateWriteApply.js"; +import { + checkDocumentRead, + checkItemRead, + checkListRead, + checkedLines, + collectionTarget, + entityTarget, + isAbsence, + itemTarget, + servedLine, + succeeded, + withBodies, + writtenFieldChecks, + SimulatedState, + type CheckOutcome, + type CheckedLines, + type ServedLine, +} from "./environmentStateWriteChecks.js"; +import { + exercisedBy, + interference, + type WriteHistory, + forgetChecked, + runningAt, + shownFields, + shownWrites, + touchTarget, + WriteEffects, + type PendingTargets, + type TouchedTargets, +} from "./environmentStateWriteAttribution.js"; +import { applyCreate } from "./environmentStateWriteCreate.js"; +import { applyDocument } from "./environmentStateWriteDocument.js"; +import { + applyDelete, + applyUpdate, + type EffectContext, + type EffectResult, + type WriteDraft, + type WriteSimulationInput, + type WriteStep, +} from "./environmentStateWriteEffects.js"; +import { insertIndex } from "./environmentStateWriteExplain.js"; +import { + classifyWrite, + prefixRelated, + unservedPlan, + type StateWritePlan, +} from "./environmentStateWriteRules.js"; +import { WriteTaint } from "./environmentStateWriteTaint.js"; +import { writeVerdict } from "./environmentStateWriteVerdict.js"; + +const APPLY = { + create: applyCreate, + update: applyUpdate, + delete: applyDelete, + document: applyDocument, +} as const; + +/** + * Replay the capture's timeline on a copy of the initial state: modelled writes apply their + * effects, other writes taint related types, and each read the served state would answer is + * compared with its recording and attributed to the writes it exercises. + */ +export class EnvironmentStateWriteSimulation { + readonly #input: WriteSimulationInput; + readonly #state: SimulatedState; + readonly #drafts = new Map(); + readonly #toolOf = new Map(); + readonly #types = new Map(); + /** Each list or document view's checked request lines, with the recording a call of the line is answered on. */ + readonly #checkedLines: ReadonlyMap; + /** Reads a write's effect was learned from, with the tools that borrowed each. */ + readonly #observations = new Map>(); + readonly #taint: WriteTaint; + readonly #touched: TouchedTargets = new Map(); + readonly #effects = new WriteEffects(); + readonly #lastCheck = new Map(); + /** When the latest modelled write to each target since its last check ended. */ + readonly #touchedSince: PendingTargets = new Map(); + /** Timeline indexes of writes the gate does not model. */ + readonly #unmodelledAt: number[] = []; + readonly #history: WriteHistory = { + touched: this.#touched, + pending: this.#touchedSince, + checked: this.#lastCheck, + unmodelledAt: this.#unmodelledAt, + }; + readonly #log: { + readonly index: number; + readonly tool: string; + readonly path: string; + }[] = []; + /** Entities a create inserted, by target, with the create's tool. */ + readonly #inserted = new Map(); + /** Creates whose insert rule a passing list check relied on, so it is no longer changed. */ + readonly #placed = new Set(); + + constructor(input: WriteSimulationInput) { + this.#input = input; + this.#state = new SimulatedState(input.entities, input.documents); + for (const type of input.types) + this.#types.set(type.collection_template, type); + for (const tool of input.tools) + for (const ordinal of tool.record_ordinals) + this.#toolOf.set(ordinal, tool); + this.#checkedLines = checkedLines(input); + this.#taint = new WriteTaint( + [...this.#types.keys()], + new Map( + input.documents.map((document) => [document.tool, document.path]), + ), + this.#state, + ); + } + + /** Every state-changing tool's plan, in manifest order, with its verdict. */ + run(): StateWritePlan[] { + const documents = new Map(); + for (const tool of this.#input.tools) + if (this.#input.views.get(tool.name)?.shape === "document") + documents.set(normalizedPath(tool.path_template), tool.name); + const types = new Set(this.#types.keys()); + for (const tool of this.#input.tools) { + if (!changesState(tool.method, tool.path_template)) continue; + const { effect, target } = classifyWrite(tool, types, documents); + this.#input.io.retain(512); + this.#drafts.set(tool.name, { + plan: { + ...unservedPlan(tool.name, effect, target, "no-state-target"), + reason: effect === "none" ? "no-state-target" : null, + }, + observed: undefined, + candidates: [], + confirmed: [], + contradictedBy: undefined, + sideEffect: false, + calls: 0, + updates: [], + sequences: new Map(), + }); + } + this.#input.records.forEach((record, index) => this.#step(record, index)); + return [...this.#drafts.values()].map((draft) => + writeVerdict(draft, this.#input), + ); + } + + #step(record: StateRecordFacts, index: number): void { + const tool = this.#toolOf.get(record.ordinal); + // A redirected or unanswered write may have changed state, but only a 2xx one shows its effect. + if (isStateWrite(record) && !succeeded(record)) + this.#unmodelled(record.path, index); + else if (isStateWrite(record)) this.#write(record, index, tool); + else if (tool?.method === "GET") this.#read(record, index, tool); + } + + #context(writer: string): EffectContext { + return { + state: this.#state, + io: this.#input.io, + records: this.#input.records, + timed: this.#input.timed, + viewOf: (ordinal) => + this.#input.views.get(this.#toolOf.get(ordinal)?.name ?? ""), + idField: (type) => this.#types.get(type)!.id_field, + observed: (ordinal) => + this.#observations.set( + ordinal, + (this.#observations.get(ordinal) ?? new Set()).add(writer), + ), + unknown: (type, collection, id) => + this.#taint.unknown(type, collection, id), + initialDocument: (tool) => + this.#input.documents.find((document) => document.tool === tool), + startedWith: (type) => + this.#input.entities.filter((entity) => entity.type === type), + }; + } + + #write( + record: StateRecordFacts, + index: number, + tool: StateToolFacts | undefined, + ): void { + const draft = tool === undefined ? undefined : this.#drafts.get(tool.name); + const effect = draft?.plan.effect; + if ( + draft === undefined || + tool === undefined || + effect === undefined || + effect === "none" + ) { + this.#unmodelled(record.path, index); + return; + } + const observeOnly = draft.plan.reason !== null; + if (!observeOnly) draft.calls += 1; + const step: WriteStep = { record, index, tool, observeOnly }; + this.#settle(APPLY[effect](this.#context(tool.name), draft, step), { + index, + tool: tool.name, + path: record.path, + end: record.end, + }); + } + + /** Attribute an applied effect to its tool; a recorded outcome is the state's new baseline; an unknown one taints. */ + #settle( + result: EffectResult, + write: { + readonly index: number; + readonly tool: string; + readonly path: string; + readonly end: number; + }, + ): void { + if (result.kind === "unknown") { + this.#unmodelled(write.path, write.index); + return; + } + if (result.entity !== undefined) + this.#taint.known( + result.entity.type, + result.entity.collection, + result.entity.id, + ); + if (result.kind === "observed") { + this.#effects.observed(this.#touched, result.targets); + // A recorded outcome replaces the entity's values, not its collection's membership. + this.#touched.delete(result.targets[0]!); + return; + } + this.#effects.supersede(this.#touched, write.tool, result); + for (const target of result.targets) + touchTarget(this.#touched, this.#touchedSince, target, { + ...write, + fields: result.fields, + }); + this.#log.push(write); + if (result.inserted !== undefined) + this.#inserted.set(result.inserted, write.tool); + } + + /** + * An item read of an entity an unmodelled write may have changed shows it again: the writes that + * changed it are checked on their own fields, and the state takes what the read recorded. + */ + #resync( + view: Extract, + record: StateRecordFacts, + entity: StateEntity, + ): void { + const target = entityTarget(entity.type, entity.collection, entity.id); + const writes = this.#touched.get(target) ?? new Map(); + this.#taint.known(entity.type, entity.collection, entity.id); + this.#touched.delete(target); + if (isAbsence(record.status)) { + this.#state.remove(entity); + return; + } + const object = this.#checkFields(view, record, entity, writes); + if (object !== undefined) + this.#state.put(updatedEntity(entity, { ...entity.fields, ...object })); + } + + /** Whether an item read is of an entity left unknown that it shows again rather than checks. */ + #resynced(view: StateView, record: StateRecordFacts): boolean { + if (view.shape !== "item") return false; + const { collection, id } = itemTarget(this.#state, view.type, record); + if (!this.#taint.unknown(view.type, collection, id)) return false; + // A read begun while a modelled write to the entity still ran may show either state, so it neither checks nor shows it. + const target = entityTarget(view.type, collection, id); + if (runningAt(this.#touchedSince, target, record.start)) return true; + // A failed read that is no absence shows nothing of the entity, so it stays unknown for a usable read. + if (!succeeded(record) && !isAbsence(record.status)) return true; + const entity = this.#state.entity(view.type, collection, id); + // A recorded absence of an entity the state no longer holds is still a check of its delete. + if (entity === undefined) { + if (isAbsence(record.status)) return false; + this.#taint.known(view.type, collection, id); + return true; + } + this.#resync(view, record, entity); + return true; + } + + #check(view: StateView, record: StateRecordFacts): CheckOutcome | undefined { + if (view.shape === "recorded") return undefined; + const line = this.#line(view, record); + if (view.shape !== "item" && line === undefined) return undefined; + if (view.shape === "document") + return this.#input.io.withBody(record.ordinal, (body) => + checkDocumentRead(this.#state, view.tool, record, body), + ); + const item = view.shape === "item"; + const collection = item + ? itemTarget(this.#state, view.type, record).collection + : line!.collection; + if (this.#taint.under(collection)) return undefined; + if (!item) { + const ids = this.#state + .members(view.type, collection) + .map((entity) => entity.id); + // A recorded-membership list covers no collection target, so a write at the collection is checked here. + if ( + this.#taint.tainted(collectionTarget(view.type, collection)) || + this.#taint.blocksList(view.type, collection, ids) + ) + return undefined; + } + // The served answer keeps the envelope of the item view's or the line's base recording. + const base = item ? view.envelope_ordinal : line!.ordinal; + const ordinals = { base, read: record.ordinal }; + return withBodies(this.#input.io.withBody, ordinals, (served, body) => { + const check = { + body, + base: served, + idField: this.#types.get(view.type)?.id_field ?? "", + volatile: this.#input.volatileOf(view.type), + collection, + }; + return item + ? checkItemRead(this.#state, view, record, check) + : checkListRead(this.#state, view, record, check); + }); + } + + /** The checked line serving answers a list or document read's call from, as replay chooses it. */ + #line(view: StateView, record: StateRecordFacts): ServedLine | undefined { + return servedLine(this.#checkedLines.get(view.tool), record); + } + + #read(record: StateRecordFacts, index: number, tool: StateToolFacts): void { + const view = this.#input.views.get(tool.name); + const borrowers = this.#observations.get(record.ordinal); + if (view === undefined || (!borrowers && this.#resynced(view, record))) + return; + const outcome = this.#check(view, record); + if (outcome === undefined) return; + const keys = [...outcome.targets, ...outcome.unheld]; + if ( + keys.some( + (key) => + this.#taint.tainted(key) || + runningAt(this.#touchedSince, key, record.start), + ) + ) + return; + const exercised = exercisedBy(this.#touched, keys, borrowers); + const interferes = interference(this.#history, outcome); + if (borrowers !== undefined) { + if (!outcome.pass) this.#contradictOthers(exercised, record, interferes); + return; + } + const pass = + outcome.pass || + (view.shape === "list" && this.#retryInsert(view, record, exercised)); + const unseen = pass + ? this.#confirm(view, record, keys, exercised) + : undefined; + if (!pass) { + const interfered = new Set([...exercised].filter(interferes)); + if (interfered.size === 0) + this.#mismatch({ record, index, keys, exercised, view }); + else { + // An unmodelled write after some of these writes may explain the difference: those are checked on their own members only. + this.#contradictOthers(exercised, record, interferes); + if (view.shape === "item") this.#checkMembers(view, record, interfered); + else for (const key of keys) this.#taint.fail(key); + } + if (view.shape === "item") this.#adopt(view, record); + } + const pending = { + touched: this.#touched, + since: this.#touchedSince, + effects: this.#effects, + }; + for (const key of keys) { + this.#lastCheck.set(key, index); + forgetChecked(pending, key, unseen?.get(key)); + } + } + + /** A failed read contradicts the writes it exercised that no unmodelled write came after, as a read a write's effect was learned from still does the others. */ + #contradictOthers( + exercised: ReadonlySet, + record: StateRecordFacts, + interferes: (name: string) => boolean, + ): void { + for (const name of exercised) + if (!interferes(name)) + this.#drafts.get(name)!.contradictedBy ??= record.ordinal; + } + + /** A passing read confirms the writes it exercised that it shows something of, and names, per target, those it shows nothing of. */ + #confirm( + view: StateView, + record: StateRecordFacts, + keys: readonly string[], + exercised: ReadonlySet, + ): Map> { + const shown = shownWrites(this.#touched, this.#effects, { + keys, + exercised, + fields: shownFields(view, this.#input.volatileOf), + }); + for (const name of shown.confirmed) { + this.#drafts.get(name)!.confirmed.push(record.ordinal); + if (view.shape === "list") this.#placed.add(name); + } + return shown.unseen; + } + + #unmodelled(path: string, index: number): void { + this.#taint.write(path); + this.#unmodelledAt.push(index); + } + + /** Check the writes to an item read's entity on the members they sent, as a resynchronising read does. */ + #checkMembers( + view: Extract, + record: StateRecordFacts, + only: ReadonlySet, + ): void { + const { collection, id } = itemTarget(this.#state, view.type, record); + const entity = this.#state.entity(view.type, collection, id); + if (entity === undefined) return; + const target = entityTarget(entity.type, entity.collection, entity.id); + const writes = [...(this.#touched.get(target) ?? [])]; + const checked = new Map(writes.filter(([name]) => only.has(name))); + this.#checkFields(view, record, entity, checked); + } + + /** Check the writes to an entity on the fields each wrote against an item read; the read's object, when it shows one. */ + #checkFields( + view: Extract, + record: StateRecordFacts, + entity: StateEntity, + writes: ReadonlyMap = new Map(), + ): JsonObject | undefined { + const object = this.#input.io.withBody(record.ordinal, (body) => + body === undefined || body === null ? undefined : at(body, view.pointer), + ); + if (!isObject(object)) return undefined; + for (const [name, pass] of writtenFieldChecks(entity, view, object, { + volatile: this.#input.volatileOf(view.type), + writes, + })) { + const draft = this.#drafts.get(name)!; + if (pass) draft.confirmed.push(record.ordinal); + else draft.contradictedBy ??= record.ordinal; + } + return object; + } + + /** After a failed item check the state takes what the read recorded, so later checks start from it. */ + #adopt( + view: Extract, + record: StateRecordFacts, + ): void { + const { collection, id } = itemTarget(this.#state, view.type, record); + const entity = this.#state.entity(view.type, collection, id); + if (entity === undefined) { + this.#taint.fail(entityTarget(view.type, collection, id)); + return; + } + const object = this.#input.io.withBody(record.ordinal, (body) => + body === undefined || body === null ? undefined : at(body, view.pointer), + ); + if (isObject(object)) + this.#state.put(updatedEntity(entity, { ...entity.fields, ...object })); + else this.#state.remove(entity); + } + + /** Contradict the writes a failed check exercised, or, when none, the related writes since its targets were last checked; a failed list stops being checked. */ + #mismatch(failed: { + readonly record: StateRecordFacts; + readonly index: number; + readonly keys: readonly string[]; + readonly exercised: ReadonlySet; + readonly view: StateView; + }): void { + if (failed.exercised.size > 0) + for (const name of failed.exercised) { + const draft = this.#drafts.get(name)!; + draft.contradictedBy ??= failed.record.ordinal; + } + else { + const since = Math.min( + ...failed.keys.map((key) => this.#lastCheck.get(key) ?? -1), + ); + for (const write of this.#log) + if ( + write.index > since && + prefixRelated(write.path, failed.record.path) + ) + this.#drafts.get(write.tool)!.sideEffect = true; + } + if (failed.view.shape !== "item") + for (const key of failed.keys) this.#taint.fail(key); + } + + /** A list that fails only for where one create placed its entities passes under another of that create's insert candidates. */ + #retryInsert( + view: Extract, + record: StateRecordFacts, + exercised: ReadonlySet, + ): boolean { + const collection = this.#line(view, record)?.collection; + if (view.membership !== "all" || collection === undefined) return false; + const members = this.#state.members(view.type, collection); + const key = (entity: StateEntity) => + entityTarget(entity.type, entity.collection, entity.id); + const inserted = members.filter((entity) => + this.#inserted.has(key(entity)), + ); + const creates = new Set( + inserted.map((entity) => this.#inserted.get(key(entity))!), + ); + if (creates.size !== 1) return false; + const [name] = creates; + const draft = this.#drafts.get(name!)!; + // An item read confirms the entity, not where a list places it. + if (!exercised.has(name!) || this.#placed.has(name!)) return false; + for (const candidate of draft.candidates.slice(1)) { + for (const entity of inserted) this.#state.remove(entity); + for (const entity of inserted) { + const order = this.#state + .members(view.type, collection) + .map((member) => member.fields); + this.#state.put(entity, insertIndex(candidate, order, entity.fields)); + } + if (this.#check(view, record)?.pass === true) { + draft.plan.insert = candidate; + return true; + } + } + return false; + } +} diff --git a/src/domain/analysisView/environmentStateGate.ts b/src/domain/analysisView/environmentStateGate.ts index 4df98d3506..0e0b175096 100644 --- a/src/domain/analysisView/environmentStateGate.ts +++ b/src/domain/analysisView/environmentStateGate.ts @@ -19,6 +19,7 @@ import { type StateToolFacts, type StateView, } from "./environmentStateRules.js"; +import { pagesOnly } from "./environmentStatePaging.js"; import { ENTRY_BYTES, textBytes } from "./webNetworkRouteTemplates.js"; /** A read tool whose recordings locate entities of one type at one pointer. */ @@ -173,8 +174,9 @@ const listFacts = ( body === undefined ? undefined : listItems(body, plan.pointer); if (body === undefined || items === undefined) return; items.forEach((item) => union.add(item)); - // Once a list cannot be the whole collection, its membership is not compared again. - if (all && record.query === undefined) { + // Once a list cannot be the whole collection, its membership is not compared again; a line + // that only pages its list can still hold all of it. + if (all && pagesOnly(record.query)) { const ids = items.map((item) => scalarText(item[field]) ?? ""); const expected = context .inCollection(plan.type, collectionPathOf(record, false)) diff --git a/src/domain/analysisView/environmentStatePaging.ts b/src/domain/analysisView/environmentStatePaging.ts new file mode 100644 index 0000000000..92b17e9ca3 --- /dev/null +++ b/src/domain/analysisView/environmentStatePaging.ts @@ -0,0 +1,62 @@ +import { + iterateUrlQueryParameters, + parseUrlQueryParameterNames, +} from "../recordedTransportUrl.js"; + +/** Query parameters that page or bound a list without filtering it. */ +const PAGING_PARAMETERS = new Set([ + "limit", + "offset", + "page", + "per_page", + "perpage", + "page_size", + "pagesize", + "size", + "start", + "_start", + "_end", +]); + +/** Whether a query only pages its list: no parameters, or paging parameters alone. */ +export const pagesOnly = (query: string | undefined): boolean => + query === undefined || + [...parseUrlQueryParameterNames(query)].every((name) => + PAGING_PARAMETERS.has(name.toLowerCase()), + ); + +/** Paging parameters that bound how many items a page holds. */ +const PAGE_SIZE_PARAMETERS = new Set([ + "limit", + "per_page", + "perpage", + "page_size", + "pagesize", + "size", +]); + +/** The most items a query's page holds: its smallest whole-number page-size value or `_start`–`_end` range width, or undefined without one; with `unboundedZero`, a size of zero is no bound. */ +const pageLimit = ( + query: string | undefined, + unboundedZero: boolean, +): number | undefined => { + let size = Infinity; + const range = { start: 0, end: Infinity }; + for (const [name, value] of iterateUrlQueryParameters(query ?? "")) { + const key = name.toLowerCase(); + if (!/^\d+$/u.test(value)) continue; + const number = Number(value); + if (PAGE_SIZE_PARAMETERS.has(key)) { + if (number > 0 || !unboundedZero) size = Math.min(size, number); + } else if (key === "_start") range.start = number; + else if (key === "_end") range.end = number; + } + const width = Math.min(size, Math.max(range.end - range.start, 0)); + return Number.isFinite(width) ? width : undefined; +}; + +/** The most items a line's call lists. A size of zero is no bound on a line whose own recording listed items under it, as an application reading zero as no limit does; every other size and range still bounds the page. */ +export const pageBound = ( + query: string | undefined, + recorded: number, +): number | undefined => pageLimit(query, recorded > 0); diff --git a/src/domain/analysisView/environmentStateRules.ts b/src/domain/analysisView/environmentStateRules.ts index e62cbda51f..d67e9edf3a 100644 --- a/src/domain/analysisView/environmentStateRules.ts +++ b/src/domain/analysisView/environmentStateRules.ts @@ -1,5 +1,6 @@ import { canonicalJson } from "../comparisonSemantics.js"; import type { JsonValue } from "../jsonValue.js"; +import type { StateWritePlan } from "./environmentStateWriteRules.js"; import { ENTRY_BYTES, textBytes } from "./webNetworkRouteTemplates.js"; /** How records are ordered: by recorded start time, or by record order when any start time is missing. */ @@ -33,6 +34,10 @@ export type ParsedBody = /** Budgeted access the accumulator lends extraction; null and false mean the budget is exhausted. */ export interface StateBudget { readonly body: (ordinal: number) => ParsedBody | null; + /** A record's request body, admitted as response bodies are. */ + readonly requestBody: (ordinal: number) => ParsedBody | null; + /** Whether a record's response had no body. */ + readonly emptyResponse: (ordinal: number) => boolean; readonly retain: (bytes: number) => boolean; /** Drop the temporary work of the body just read. */ readonly release: () => void; @@ -125,6 +130,9 @@ export interface ExtractedState { readonly entities: StateEntity[]; readonly documents: StateDocument[]; readonly views: StateView[]; + readonly writes: StateWritePlan[]; + /** The instant a served clock starts from: the end of the capture's last timed record, or null without times. */ + readonly write_clock: string | null; } const WRITE_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]); @@ -181,9 +189,8 @@ export const same = ( : canonicalJson(left, "Environment state") === canonicalJson(right, "Environment state"); -/** The decoded final path segment, which an item request names its entity by. */ -export const lastSegmentValue = (path: string): string => { - const segment = segmentsOf(path).at(-1) ?? ""; +/** A path segment percent-decoded, or as written when it does not decode. */ +export const decodedSegment = (segment: string): string => { try { return decodeURIComponent(segment); } catch { @@ -191,6 +198,10 @@ export const lastSegmentValue = (path: string): string => { } }; +/** The decoded final path segment, which an item request names its entity by. */ +export const lastSegmentValue = (path: string): string => + decodedSegment(segmentsOf(path).at(-1) ?? ""); + /** A path's non-empty segments, so a trailing slash names the same collection. */ export const normalizedPath = (path: string): string => `/${segmentsOf(path).join("/")}`; @@ -509,6 +520,16 @@ export const renderStateEntity = ( ); }; +/** The fields every recording of a view carried that the entity, as serving renders it, lacks; serving refuses to answer with such an entity. */ +export const missingRequiredFields = ( + entity: StateEntity, + view: Pick, + volatileFields: ReadonlySet, +): string[] => { + const shown = renderStateEntity(entity, view, volatileFields); + return view.required_fields.filter((name) => !Object.hasOwn(shown, name)); +}; + /** A list response rendered from `base`: its array replaced by `items` and its counters set to their number. */ export const renderStateList = ( view: Pick, diff --git a/src/domain/analysisView/environmentStateView.ts b/src/domain/analysisView/environmentStateView.ts index fd8273c7dc..dcaf204c90 100644 --- a/src/domain/analysisView/environmentStateView.ts +++ b/src/domain/analysisView/environmentStateView.ts @@ -87,6 +87,131 @@ const viewSchema = z.discriminatedUnion("shape", [ }), ]); +const ordinalSchema = z.number().int().nonnegative(); +const clockFormatSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("iso"), + separator: z.enum(["T", " "]), + fraction: z.number().int().nonnegative(), + zone: z.string(), + }), + z.strictObject({ kind: z.literal("epoch"), unit: z.enum(["s", "ms"]) }), +]); +const ruled = { field: z.string() }; +const fieldRuleSchema = z.discriminatedUnion("rule", [ + z.strictObject({ + ...ruled, + rule: z.literal("identity"), + generator: z.enum(["sequence", "uuid", "token"]), + }), + z.strictObject({ + ...ruled, + rule: z.literal("sequence"), + scope: z.enum(["collection", "type"]), + }), + z.strictObject({ ...ruled, rule: z.literal("unique") }), + z.strictObject({ + ...ruled, + rule: z.literal("clock"), + format: clockFormatSchema, + }), + z.strictObject({ ...ruled, rule: z.literal("echo") }), + z.strictObject({ ...ruled, rule: z.literal("copy"), from: z.string() }), + z.strictObject({ ...ruled, rule: z.literal("path"), from: z.string() }), + z.strictObject({ ...ruled, rule: z.literal("slug"), from: z.string() }), + z.strictObject({ ...ruled, rule: z.literal("template"), from: z.string() }), + z.strictObject({ ...ruled, rule: z.literal("increment") }), + z.strictObject({ + ...ruled, + rule: z.literal("constant"), + value: jsonValueSchema, + }), +]); +const insertRuleSchema = z.union([ + z.strictObject({ rule: z.enum(["first", "last"]) }), + z.strictObject({ + rule: z.literal("sorted"), + field: z.string(), + direction: z.enum(["ascending", "descending"]), + }), +]); +const writePlanSchema = z.strictObject({ + tool: z.string(), + effect: z.enum(["create", "update", "delete", "document", "none"]), + target: z + .string() + .nullable() + .describe( + "The entity type an entity effect writes, or the document view a document write changes.", + ), + served: z.boolean(), + reason: z + .enum([ + "no-state-target", + "no-call", + "body-unusable", + "no-observation", + "unexplained-change", + "contradicted", + "side-effect", + "unconfirmed", + "incomplete-entities", + ]) + .nullable(), + exemplar_ordinal: ordinalSchema + .nullable() + .describe("The first successful recorded call the plan was built from."), + observation: z + .strictObject({ ordinal: ordinalSchema, pointer: z.string() }) + .nullable() + .describe( + "The record whose response showed the written entity or document, and a JSON pointer to it.", + ), + response: z + .strictObject({ + ordinal: ordinalSchema, + shape: z.enum(["entity", "identity", "empty", "recorded"]), + pointer: z.string().nullable(), + }) + .nullable(), + body: z + .strictObject({ + accepted: z.array(z.string()), + fixed: z.array( + z.strictObject({ member: z.string(), sent: jsonValueSchema }), + ), + required: z.array(z.string()), + identity: z.string().nullable(), + kinds: z.array( + z.strictObject({ + member: z.string(), + kind: z.enum(["array", "boolean", "number", "object", "string"]), + }), + ), + }) + .nullable() + .describe( + "Body members the application stored as sent (accepted), members accepted only with their recorded value (fixed), members every call sends (required), and the JSON type a later call stored an accepted member as when the observation does not show it (kinds).", + ), + rules: z + .array(fieldRuleSchema) + .describe( + "How written fields get their values; unlisted fields of a created entity keep the observed value.", + ), + collection: z + .string() + .nullable() + .describe( + "For a create whose recorded values are tied to its collection, the only collection it writes to.", + ), + insert: insertRuleSchema.nullable(), + merge: z.enum(["merge", "replace"]).nullable(), + checked_record_ordinals: z + .array(ordinalSchema) + .describe("Later reads that reproduced from the state after this write."), + contradicted_by: ordinalSchema.nullable(), +}); + export const environmentStateFields = { origin: z .string() @@ -146,6 +271,17 @@ export const environmentStateFields = { }), ), views: z.array(viewSchema), + writes: z + .array(writePlanSchema) + .describe( + "One plan per tool whose method may change state: its modelled effect, and whether the capture's later reads confirmed it (served).", + ), + write_clock: z + .string() + .nullable() + .describe( + "The instant a served clock starts from: the end of the capture's last timed record.", + ), unknowns: z.array( z.strictObject({ kind: z.enum([ diff --git a/src/domain/analysisView/environmentStateWriteApply.test.ts b/src/domain/analysisView/environmentStateWriteApply.test.ts new file mode 100644 index 0000000000..3ea99ac31e --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteApply.test.ts @@ -0,0 +1,534 @@ +import { expect, it } from "vitest"; +import type { JsonValue } from "../jsonValue.js"; +import { err, ok } from "../result.js"; +import type { JsonObject, StateEntity } from "./environmentStateRules.js"; +import { + bodyProblem, + createdFields, + updatedEntity, + updatedFields, + valueAt, + writtenDocument, + writtenResponse, +} from "./environmentStateWriteApply.js"; +import type { + FieldRule, + GeneratedRule, + StateWritePlan, + WriteBody, + WriteCall, + WriteResponse, +} from "./environmentStateWriteRules.js"; +import type { ClockFormat } from "./environmentStateWriteValues.js"; + +const SECONDS: ClockFormat = { + kind: "iso", + separator: "T", + fraction: 0, + zone: "Z", +}; +const NOW = "2026-03-04T05:06:07Z"; + +const callWith = ( + body: JsonObject | undefined, + pathValues: Record = {}, +): WriteCall => ({ + path: "/items", + pathValues: new Map(Object.entries(pathValues)), + body, +}); + +/** Generated values the store would supply, recording which rules asked. */ +const generator = () => { + const asked: GeneratedRule[] = []; + const values = { identity: "id-9", sequence: 9, unique: "ref-9", clock: NOW }; + const generate = (rule: GeneratedRule) => { + asked.push(rule); + return ok(values[rule.rule]); + }; + return { asked, generate }; +}; + +const CONTRACT: WriteBody = { + accepted: ["title", "tags", "meta", "note"], + fixed: [ + { member: "priority", sent: "HIGH" }, + { member: "options", sent: { a: 1, b: 2 } }, + ], + required: ["title"], + identity: "id", + kinds: [], +}; +const RECORDED = { title: "A", tags: ["x"], meta: null, priority: "high" }; +const LISTED = + 'accepted members: "title", "tags", "meta", "note"; members accepted only with their recorded value: "priority", "options"; required: "title"'; + +const problemOf = (body: JsonObject | undefined): string | undefined => + bodyProblem(CONTRACT, callWith(body), { entity: RECORDED, id: "42" }); + +it.each<[string, JsonObject]>([ + [ + "accepted and fixed members", + { title: "B", priority: "HIGH", options: { b: 2, a: 1 } }, + ], + ["the identity repeated", { id: "42", title: "B" }], + ["the identity repeated as a number", { id: 42, title: "B" }], + ["any type where the recorded value is null", { title: "B", meta: { x: 1 } }], + ["any type where no value was recorded", { title: "B", note: 5 }], +])("accepts a body with %s", (_, body) => { + expect(problemOf(body)).toBeUndefined(); +}); + +it.each<[string, JsonObject | undefined, string]>([ + [ + "an unknown member", + { title: "B", color: "red" }, + `Member color is not one the capture showed stored (${LISTED}).`, + ], + [ + "a fixed member with another value", + { title: "B", priority: "LOW" }, + `Member priority is accepted only with the value the capture sent, because the application stored it differently (${LISTED}).`, + ], + [ + "a number for a string", + { title: 5 }, + "Member title must be a JSON string, as the capture stored it.", + ], + [ + "null for a string", + { title: null }, + "Member title must be a JSON string, as the capture stored it.", + ], + [ + "an object for an array", + { title: "B", tags: { x: true } }, + "Member tags must be a JSON array, as the capture stored it.", + ], + [ + "a missing required member", + { tags: ["y"] }, + `The call lacks required members "title" (${LISTED}).`, + ], + [ + "no body", + undefined, + `The call lacks required members "title" (${LISTED}).`, + ], + [ + "another identity", + { id: "43", title: "B" }, + "Member id must repeat the target's identity 42.", + ], +])("refuses a body with %s", (_, body, message) => { + expect(problemOf(body)).toBe(message); +}); + +it.each<[string, JsonValue | undefined]>([ + ["", { a: [{ b: 1 }, { b: [2, 3] }], "x/y": { "m~n": true }, "": "blank" }], + ["/a/0/b", 1], + ["/a/1/b/1", 3], + ["/x~1y/m~0n", true], + ["/", "blank"], + ["/a/2/b", undefined], + ["/a/first", undefined], + ["/a/0/b/c", undefined], + ["/missing", undefined], +])("reads the body value at %j", (pointer, expected) => { + const body = { + a: [{ b: 1 }, { b: [2, 3] }], + "x/y": { "m~n": true }, + "": "blank", + }; + expect(valueAt(body, pointer)).toEqual(expected); +}); + +const CREATE_RULES: FieldRule[] = [ + { field: "id", rule: "identity", generator: "token" }, + { field: "number", rule: "sequence", scope: "collection" }, + { field: "createdAt", rule: "clock", format: SECONDS }, + { field: "ref", rule: "unique" }, + { field: "title", rule: "echo" }, + { field: "summary", rule: "copy", from: "/meta/notes/1" }, + { field: "projectId", rule: "path", from: "project" }, + { field: "projectKey", rule: "path", from: "project" }, + { field: "slug", rule: "slug", from: "title" }, + { field: "url", rule: "template", from: "id" }, +]; +type Plan = Pick; + +const CREATE_PLAN: Plan = { + rules: CREATE_RULES, + body: { + accepted: ["title", "meta"], + fixed: [], + required: ["title", "meta"], + identity: null, + kinds: [], + }, +}; +const CREATE_OBSERVED: JsonObject = { + id: "id-1", + number: 1, + createdAt: "2026-01-01T00:00:00Z", + ref: "ref-1", + title: "Old title", + summary: "old summary", + projectId: 7, + projectKey: "7", + slug: "old-title", + url: "https://x.test/items/id-1?v=id-10", + status: "open", +}; +const NEW_ITEM = { + title: "New Title!", + meta: { notes: ["a", "the new summary"] }, +}; + +it("creates an entity from generated values, the call and the observed constants", () => { + const { asked, generate } = generator(); + const observed = structuredClone(CREATE_OBSERVED); + const result = createdFields( + CREATE_PLAN, + observed, + callWith(NEW_ITEM, { project: "42" }), + generate, + ); + expect(result).toEqual( + ok({ + id: "id-9", + number: 9, + createdAt: NOW, + ref: "ref-9", + title: "New Title!", + summary: "the new summary", + projectId: 42, + projectKey: "42", + slug: "new-title", + url: "https://x.test/items/id-9?v=id-10", + status: "open", + }), + ); + expect(asked).toEqual(CREATE_RULES.slice(0, 4)); + expect(observed).toEqual(CREATE_OBSERVED); +}); + +it.each<[string, Plan, WriteCall, string]>([ + [ + "no modelled body", + { rules: CREATE_RULES, body: null }, + callWith(NEW_ITEM, { project: "42" }), + "The tool has no modelled body.", + ], + [ + "a missing copy source", + CREATE_PLAN, + callWith({ title: "x", meta: { notes: ["only one"] } }, { project: "42" }), + "The call has no value at /meta/notes/1 in its body, which summary copies.", + ], + [ + "a missing path value", + CREATE_PLAN, + callWith(NEW_ITEM), + "The call names no project path value, which projectId takes.", + ], + [ + "a body problem", + CREATE_PLAN, + callWith({ ...NEW_ITEM, extra: 1 }, { project: "42" }), + 'Member extra is not one the capture showed stored (accepted members: "title", "meta"; members accepted only with their recorded value: none; required: "title", "meta").', + ], +])("refuses a create with %s", (_, plan, call, message) => { + expect( + createdFields(plan, CREATE_OBSERVED, call, generator().generate), + ).toEqual(err(message)); +}); + +const UPDATE_PLAN: Plan = { + rules: [ + { field: "title", rule: "echo" }, + { field: "notes", rule: "echo" }, + { field: "status", rule: "constant", value: "closed" }, + { field: "slug", rule: "slug", from: "title" }, + { field: "version", rule: "increment" }, + { field: "revision", rule: "increment" }, + { field: "updatedAt", rule: "clock", format: SECONDS }, + ], + body: { + accepted: ["title", "notes"], + fixed: [{ member: "status", sent: "CLOSED" }], + required: [], + identity: "id", + kinds: [], + }, +}; +const BEFORE: JsonObject = { + id: "42", + title: "Old Title", + notes: "n", + status: "open", + slug: "old-title", + version: 3, + revision: "9", + updatedAt: "2026-01-01T00:00:00Z", + color: "red", +}; +const AFTER: JsonObject = { + ...BEFORE, + title: "Recorded Title", + status: "closed", + slug: "recorded-title", + version: 4, + revision: "10", +}; + +const update = (body: JsonObject | undefined, plan = UPDATE_PLAN) => { + const { asked, generate } = generator(); + const result = updatedFields( + plan, + { before: BEFORE, observed: AFTER }, + { ...callWith(body, { item: "42" }), id: "42" }, + generate, + ); + return { asked, result }; +}; + +const ADVANCED = { version: 4, revision: "10", updatedAt: NOW }; + +it("updates sent echoed members and derives the slug of a sent member", () => { + const { asked, result } = update({ id: "42", title: "New Title" }); + expect(result).toEqual( + ok({ ...BEFORE, ...ADVANCED, title: "New Title", slug: "new-title" }), + ); + expect(asked).toEqual([UPDATE_PLAN.rules[6]]); +}); + +it("preserves an unknown generated field when the observation omits it", () => { + const result = updatedFields( + UPDATE_PLAN, + { before: BEFORE, observed: AFTER }, + { ...callWith({ title: "New Title" }, { item: "42" }), id: "42" }, + () => ok(undefined), + ); + expect(result).toEqual( + ok({ + ...BEFORE, + title: "New Title", + slug: "new-title", + version: 4, + revision: "10", + }), + ); +}); + +it("applies a constant from the recorded update only when the call sends it", () => { + expect(update({ status: "CLOSED" }).result).toEqual( + ok({ ...BEFORE, ...ADVANCED, status: "closed" }), + ); + expect(update({ notes: "m" }).result).toEqual( + ok({ ...BEFORE, ...ADVANCED, notes: "m" }), + ); +}); + +it.each<[string, JsonObject | undefined, string]>([ + [ + "only the identity", + { id: "42" }, + "The call sends none of the members an update applies", + ], + ["no members", {}, "The call sends none of the members an update applies"], + [ + "no body", + undefined, + "The call sends none of the members an update applies", + ], + [ + "another identity", + { id: "43", title: "x" }, + "Member id must repeat the target's identity 42.", + ], + ["a number for a string", { title: 5 }, "Member title must be a JSON string"], + [ + "a fixed member with another value", + { status: "OPEN" }, + "Member status is accepted only with the value", + ], +])("refuses an update with %s", (_, body, message) => { + const { result } = update(body); + expect(result.ok ? undefined : result.error).toContain(message); +}); + +it("refuses an update without a modelled body", () => { + expect(update({ title: "x" }, { ...UPDATE_PLAN, body: null }).result).toEqual( + err("The tool has no modelled body."), + ); +}); + +const DOCUMENT = { settings: { theme: "dark", lang: "en" }, version: 1 }; + +const documentPlan = ( + merge: "merge" | "replace", + pointer?: string, +): Pick => ({ + merge, + observation: pointer === undefined ? null : { ordinal: 1, pointer }, +}); + +it.each<[string, ReturnType, JsonObject, JsonValue]>([ + [ + "merges into a member", + documentPlan("merge", "/settings"), + { theme: "light" }, + { settings: { theme: "light", lang: "en" }, version: 1 }, + ], + [ + "replaces a member", + documentPlan("replace", "/settings"), + { theme: "light" }, + { settings: { theme: "light" }, version: 1 }, + ], + [ + "merges into the root", + documentPlan("merge"), + { version: 2 }, + { ...DOCUMENT, version: 2 }, + ], + [ + "replaces the root", + documentPlan("replace", ""), + { theme: "x" }, + { theme: "x" }, + ], +])("%s of a document", (_, plan, body, expected) => { + expect(writtenDocument(plan, DOCUMENT, callWith(body))).toEqual(ok(expected)); + expect(DOCUMENT).toEqual({ + settings: { theme: "dark", lang: "en" }, + version: 1, + }); +}); + +it("refuses a document write that does not reach an object", () => { + const refused = err("The call's body does not fit the document."); + const plan = (pointer: string) => documentPlan("merge", pointer); + expect( + writtenDocument(plan("/version"), DOCUMENT, callWith({ a: 1 })), + ).toEqual(refused); + expect( + writtenDocument(plan("/settings"), DOCUMENT, callWith(undefined)), + ).toEqual(refused); +}); + +const answer = ( + shape: WriteResponse["shape"], + pointer: string | null, +): WriteResponse => ({ ordinal: 1, shape, pointer }); +const WRITTEN = { id: "8", title: "New", extra: true }; + +it.each<[string, WriteResponse, JsonValue | undefined, JsonValue | undefined]>([ + ["no body", answer("empty", null), { id: "1" }, undefined], + ["the recorded body", answer("recorded", null), { ok: true }, { ok: true }], + [ + "no body when none was recorded", + answer("entity", ""), + undefined, + undefined, + ], + ["a numeric identity at the root", answer("identity", ""), 7, 8], + ["a string identity at the root", answer("identity", ""), "7", "8"], + [ + "a numeric identity in a member", + answer("identity", "/id"), + { id: 7, ok: true }, + { id: 8, ok: true }, + ], + [ + "the recorded fields of the entity", + answer("entity", "/data"), + { data: { title: "Old", id: "1" }, page: 1 }, + { data: { title: "New", id: "8" }, page: 1 }, + ], + [ + "the entity in a one-element array", + answer("entity", "/0"), + [{ id: "1" }], + [{ id: "8" }], + ], +])("answers with %s", (_, response, recorded, expected) => { + expect( + writtenResponse(response, recorded, { fields: WRITTEN, id: "8" }), + ).toEqual(ok(expected)); +}); + +it("spells an identity that is not an integer as a string where a number was recorded", () => { + expect( + writtenResponse( + answer("identity", "/id"), + { id: 7 }, + { fields: undefined, id: "abc" }, + ), + ).toEqual(ok({ id: "abc" })); +}); + +it("refuses an entity answer the state cannot render", () => { + const recorded = { data: { id: "1", title: "Old", owner: "x" } }; + expect( + writtenResponse(answer("entity", "/data"), recorded, { + fields: WRITTEN, + id: "8", + }), + ).toEqual( + err( + "The state holds no complete entity for this response: it lacks owner.", + ), + ); + expect( + writtenResponse(answer("entity", "/data"), recorded, { + fields: undefined, + id: "8", + }), + ).toEqual(err("The recorded response does not carry the entity.")); + expect( + writtenResponse(answer("entity", "/missing"), recorded, { + fields: WRITTEN, + id: "8", + }), + ).toEqual(err("The recorded response does not carry the entity.")); +}); + +it("drops view-specific values of the fields an update changed and cites no records", () => { + const entity: StateEntity = { + type: "/items", + collection: "/items", + id: "1", + fields: { id: "1", title: "Old", status: "open", meta: { a: 1, b: 2 } }, + view_fields: [ + { + tool: "list_items", + fields: { title: "Old…", status: "OPEN", meta: "a,b" }, + }, + { tool: "get_item", fields: { title: "OLD" } }, + { tool: "search_items", fields: { priority: "HIGH" } }, + ], + record_ordinals: [3, 5], + }; + const snapshot = structuredClone(entity); + const fields = { + id: "1", + title: "New", + status: "open", + meta: { b: 2, a: 1 }, + priority: "low", + }; + expect(updatedEntity(entity, fields)).toEqual({ + ...snapshot, + fields, + record_ordinals: [], + view_fields: [ + { tool: "list_items", fields: { status: "OPEN", meta: "a,b" } }, + ], + }); + expect(entity).toEqual(snapshot); + expect(updatedEntity(entity, entity.fields).view_fields).toEqual( + entity.view_fields, + ); +}); diff --git a/src/domain/analysisView/environmentStateWriteApply.ts b/src/domain/analysisView/environmentStateWriteApply.ts new file mode 100644 index 0000000000..92eaf9a64e --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteApply.ts @@ -0,0 +1,381 @@ +import type { JsonValue } from "../jsonValue.js"; +import { err, ok, type Result } from "../result.js"; +import { + at, + isObject, + pointerTokens, + replaceAt, + same, + scalarText, + type JsonObject, + type StateEntity, +} from "./environmentStateRules.js"; +import type { + FieldRule, + GeneratedRule, + JsonKind, + StateWritePlan, + WriteBody, + WriteCall, + WriteResponse, +} from "./environmentStateWriteRules.js"; +import { + integerOf, + replaceDelimited, + slugOf, + spelledLike, +} from "./environmentStateWriteValues.js"; + +/** A refusal message naming what the call did wrong; the state is left unchanged. */ +type Applied = Result; + +/** A generated value or a typed refusal when the store cannot safely provide one. */ +export type GenerateValue = ( + rule: GeneratedRule, +) => Applied; + +const kindOf = (value: JsonValue | undefined): string => + value === null ? "null" : Array.isArray(value) ? "array" : typeof value; + +/** The JSON types `stored` holds members in, for those `observed` shows no value of. */ +export const storedKinds = ( + members: readonly string[], + observed: JsonObject, + stored: JsonObject, +): WriteBody["kinds"] => + members.flatMap((member) => { + const value = stored[member]; + return (observed[member] ?? null) === null && + value !== undefined && + value !== null + ? [{ member, kind: kindOf(value) as JsonKind }] + : []; + }); + +/** Members a refusal names, each cut to `LISTED_NAME_CHARS`; the plan in the state Evidence holds every one whole. */ +const LISTED_MEMBERS = 12; +const LISTED_NAME_CHARS = 120; + +/** Captured member names quoted, so a separator in one cannot read as another, and bounded in count and length. */ +const listed = (members: readonly string[]): string => { + if (members.length === 0) return "none"; + const shown = members + .slice(0, LISTED_MEMBERS) + .map((member) => + JSON.stringify( + member.length > LISTED_NAME_CHARS + ? `${member.slice(0, LISTED_NAME_CHARS)}…` + : member, + ), + ) + .join(", "); + return members.length > LISTED_MEMBERS + ? `${shown}, … (${String(members.length)} members; all in the state Evidence)` + : shown; +}; + +/** What a body may hold, for refusal messages. */ +const bodyContract = (body: WriteBody): string => + `accepted members: ${listed(body.accepted)}; members accepted only with their recorded value: ${listed(body.fixed.map(({ member }) => member))}; required: ${listed(body.required)}`; + +/** Why a call's body does not fit the members the capture showed, if it does not; `shapes` gives each accepted member's JSON type, else the body's recorded kind. */ +export const bodyProblem = ( + body: WriteBody, + call: WriteCall, + shapes: { readonly entity: JsonObject; readonly id?: string }, +): string | undefined => { + const sent = call.body ?? {}; + const fixed = new Map(body.fixed.map((entry) => [entry.member, entry.sent])); + const kinds = new Map(body.kinds.map((entry) => [entry.member, entry.kind])); + for (const [member, value] of Object.entries(sent)) { + if (member === body.identity) { + if (scalarText(value) !== shapes.id) + return `Member ${member} must repeat the target's identity ${shapes.id ?? ""}.`; + continue; + } + if (fixed.has(member)) { + if (!same(fixed.get(member), value)) + return `Member ${member} is accepted only with the value the capture sent, because the application stored it differently (${bodyContract(body)}).`; + continue; + } + if (!body.accepted.includes(member)) + return `Member ${member} is not one the capture showed stored (${bodyContract(body)}).`; + const recorded = shapes.entity[member]; + const kind = + recorded === undefined || recorded === null + ? kinds.get(member) + : kindOf(recorded); + if (kind !== undefined && kind !== kindOf(value)) + return `Member ${member} must be a JSON ${kind}, as the capture stored it.`; + } + const missing = body.required.filter( + (member) => !Object.hasOwn(sent, member), + ); + if (missing.length > 0) + return `The call lacks required members ${listed(missing)} (${bodyContract(body)}).`; + return undefined; +}; + +/** The value at a JSON pointer inside a body, following array indexes and members. */ +export const valueAt = ( + body: JsonValue, + pointer: string, +): JsonValue | undefined => { + let value: JsonValue | undefined = body; + for (const token of pointerTokens(pointer)) { + if (Array.isArray(value)) value = value[Number(token)]; + else if (isObject(value) && Object.hasOwn(value, token)) + value = value[token]; + else return undefined; + } + return value; +}; + +/** A path value spelled as the observed value: a number when the observed value is one. */ +const pathValue = (observed: JsonValue | undefined, text: string): JsonValue => + typeof observed === "number" && integerOf(text) !== undefined + ? Number(text) + : text; + +/** One field's value under a rule that reads the call or generates. */ +const ruledValue = ( + rule: FieldRule, + observed: JsonObject, + call: WriteCall, + generate: GenerateValue, +): Applied => { + const body = call.body ?? {}; + switch (rule.rule) { + case "identity": + case "sequence": + case "unique": + case "clock": + return generate(rule); + case "echo": + return ok(Object.hasOwn(body, rule.field) ? body[rule.field] : undefined); + case "copy": { + const value = valueAt(body, rule.from); + return value === undefined + ? err( + `The call has no value at ${rule.from} in its body, which ${rule.field} copies.`, + ) + : ok(value); + } + case "path": { + const value = call.pathValues.get(rule.from); + return value === undefined + ? err( + `The call names no ${rule.from} path value, which ${rule.field} takes.`, + ) + : ok(pathValue(observed[rule.field], value)); + } + default: + return ok(undefined); + } +}; + +/** A string field rebuilt around the new value of the generated field it embeds. */ +const templated = ( + observed: JsonObject, + written: JsonObject, + rule: Extract, +): JsonValue | undefined => { + const text = observed[rule.field]; + const token = scalarText(observed[rule.from]); + const replacement = scalarText(written[rule.from]); + return typeof text === "string" && + token !== undefined && + replacement !== undefined + ? replaceDelimited(text, token, replacement) + : undefined; +}; + +/** Apply the rules that derive from fields already written: slugs and templates. */ +const derive = ( + rules: readonly FieldRule[], + observed: JsonObject, + written: JsonObject, + sent: JsonObject, +): void => { + for (const rule of rules) { + if (rule.rule === "slug") { + const from = sent[rule.from]; + if (typeof from === "string") written[rule.field] = slugOf(from); + } else if (rule.rule === "template") { + const value = templated(observed, written, rule); + if (value !== undefined) written[rule.field] = value; + } + } +}; + +/** The fields of a created entity: the observed entity with every ruled field taken from the call or generated. */ +export const createdFields = ( + plan: Pick, + observed: JsonObject, + call: WriteCall, + generate: GenerateValue, +): Applied => { + if (plan.body === null) return err("The tool has no modelled body."); + const problem = bodyProblem(plan.body, call, { entity: observed }); + if (problem !== undefined) return err(problem); + const written: JsonObject = { ...observed }; + for (const rule of plan.rules) { + const value = ruledValue(rule, observed, call, generate); + if (!value.ok) return value; + if (value.value !== undefined) written[rule.field] = value.value; + else if (rule.rule === "echo") delete written[rule.field]; + } + derive(plan.rules, observed, written, call.body ?? {}); + return ok(written); +}; + +/** The fields of an updated entity: sent echoed members, the constants the plan recorded, generated and derived changes. */ +export const updatedFields = ( + plan: Pick, + entity: { readonly before: JsonObject; readonly observed: JsonObject }, + call: WriteCall & { readonly id: string }, + generate: GenerateValue, +): Applied => { + const { before, observed } = entity; + if (plan.body === null) return err("The tool has no modelled body."); + const sent = call.body ?? {}; + const problem = bodyProblem(plan.body, call, { + entity: { ...observed, ...before }, + id: call.id, + }); + if (problem !== undefined) return err(problem); + const members = Object.keys(sent).filter( + (member) => member !== plan.body?.identity, + ); + if (members.length === 0) + return err( + `The call sends none of the members an update applies (${bodyContract(plan.body)}).`, + ); + const written: JsonObject = { ...before }; + for (const rule of plan.rules) { + if (rule.rule === "echo" || rule.rule === "constant") { + if (!Object.hasOwn(sent, rule.field)) continue; + written[rule.field] = + rule.rule === "echo" ? sent[rule.field]! : rule.value; + } else if (rule.rule === "increment") { + const was = integerOf(before[rule.field]); + if (was !== undefined) + written[rule.field] = spelledLike(before[rule.field]!, was + 1); + } else if (rule.rule === "clock") { + const value = generate(rule); + if (!value.ok) return value; + if (value.value !== undefined) written[rule.field] = value.value; + } + } + derive( + plan.rules.filter( + (rule) => rule.rule !== "slug" || Object.hasOwn(sent, rule.from), + ), + observed, + written, + sent, + ); + return ok(written); +}; + +/** Why a document write's body does not fit what the capture showed: members its plan accepts, as the JSON types the document holds, a member a replacing write dropped keeping the type the initial document showed. */ +export const documentBodyProblem = ( + plan: Pick, + call: WriteCall, + documents: { + readonly initial: JsonValue | undefined; + readonly current: JsonValue; + }, +): string | undefined => { + if (plan.body === null) return "The tool has no modelled body."; + const pointer = plan.observation?.pointer ?? ""; + const members = (value: JsonValue | undefined): JsonObject => { + const object = value === undefined ? undefined : at(value, pointer); + return isObject(object) ? object : {}; + }; + return bodyProblem(plan.body, call, { + entity: { ...members(documents.initial), ...members(documents.current) }, + }); +}; + +/** A document after a write: the body's members merged into, or replacing, the object at the pointer. */ +export const writtenDocument = ( + plan: Pick, + before: JsonValue, + call: WriteCall, +): Applied => { + const pointer = plan.observation?.pointer ?? ""; + const target = at(before, pointer); + if (!isObject(target) || call.body === undefined) + return err("The call's body does not fit the document."); + const value = + plan.merge === "replace" ? { ...call.body } : { ...target, ...call.body }; + return ok(replaceAt(before, pointer, value)); +}; + +/** A write's answer body from its recorded response: the entity or identity rendered in, nothing, or the recorded body. */ +export const writtenResponse = ( + response: WriteResponse, + recorded: JsonValue | undefined, + written: { readonly fields: JsonObject | undefined; readonly id: string }, +): Applied => { + if (response.shape === "empty") return ok(undefined); + if (response.shape === "recorded" || recorded === undefined) + return ok(recorded); + const pointer = response.pointer ?? ""; + const shown = at(recorded, pointer); + if (response.shape === "identity") + return ok( + replaceAt( + recorded, + pointer, + typeof shown === "number" && integerOf(written.id) !== undefined + ? Number(written.id) + : written.id, + ), + ); + if (!isObject(shown) || written.fields === undefined) + return err("The recorded response does not carry the entity."); + const missing = Object.keys(shown).filter( + (field) => !Object.hasOwn(written.fields!, field), + ); + if (missing.length > 0) + return err( + `The state holds no complete entity for this response: it lacks ${missing.join(", ")}.`, + ); + return ok( + replaceAt( + recorded, + pointer, + Object.fromEntries( + Object.keys(shown).map((field) => [field, written.fields![field]!]), + ), + ), + ); +}; + +/** An entity with updated fields: view-specific values of the fields that changed give way to the new values, and it cites no records. */ +export const updatedEntity = ( + entity: StateEntity, + fields: JsonObject, +): StateEntity => { + const changed = new Set( + Object.keys(fields).filter( + (field) => !same(entity.fields[field], fields[field]), + ), + ); + // Written values are no longer the ones the cited records show. + return { + ...entity, + fields, + record_ordinals: [], + view_fields: entity.view_fields + .map((view) => ({ + tool: view.tool, + fields: Object.fromEntries( + Object.entries(view.fields).filter(([field]) => !changed.has(field)), + ), + })) + .filter((view) => Object.keys(view.fields).length > 0), + }; +}; diff --git a/src/domain/analysisView/environmentStateWriteAttribution.ts b/src/domain/analysisView/environmentStateWriteAttribution.ts new file mode 100644 index 0000000000..7b20d4ea10 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteAttribution.ts @@ -0,0 +1,350 @@ +import type { StateView } from "./environmentStateRules.js"; + +/** The fields a read of a view shows of its entities, less volatile ones it does not compare; all of a document's. */ +export const shownFields = ( + view: StateView, + volatileOf: (type: string) => ReadonlySet, +): ReadonlySet | "all" => { + if (view.shape !== "item" && view.shape !== "list") return "all"; + const volatile = volatileOf(view.type); + return new Set(view.fields.filter((field) => !volatile.has(field))); +}; + +/** Modelled writes per target since its last check, with the fields each wrote (null: all). */ +export type TouchedTargets = Map>; + +/** Per target, when the latest modelled write since its last check ended, and each tool's latest call there. */ +export type PendingTargets = Map< + string, + { until: number; readonly at: Map } +>; + +/** Whether a write that ended at `until` was still running when a read began at `start`, so the read may show either state. */ +export const overlapsWrite = (until: number, start: number): boolean => + until > start; + +/** What the gate tracks of the writes since each target's last check, and when each unmodelled write ran. */ +export interface WriteHistory { + readonly touched: TouchedTargets; + readonly pending: PendingTargets; + readonly checked: ReadonlyMap; + readonly unmodelledAt: readonly number[]; +} + +/** The tools a failed read's difference an unmodelled write may explain: it came after the tool's call on a target where its effect survives, and after every differing target's last check and pending writes. */ +export const interference = ( + history: WriteHistory, + read: { + readonly targets: readonly string[]; + readonly unheld: readonly string[]; + readonly differing?: readonly string[]; + }, +): ((tool: string) => boolean) => { + const keys = [...read.targets, ...read.unheld]; + const after = (since: number) => + history.unmodelledAt.some((index) => index > since); + const explained = (read.differing ?? []).every((key) => + after( + Math.max( + history.checked.get(key) ?? -1, + ...(history.pending.get(key)?.at.values() ?? []), + ), + ), + ); + return (tool) => + explained && + keys.some((key) => { + if (history.touched.get(key)?.has(tool) !== true) return false; + const at = history.pending.get(key)?.at.get(tool); + return at !== undefined && after(at); + }); +}; + +/** Whether a modelled write to the target was still running when a read began at `start`, so the read may show either state. */ +export const runningAt = ( + pending: PendingTargets, + key: string, + start: number, +): boolean => overlapsWrite(pending.get(key)?.until ?? -Infinity, start); + +/** The tools whose writes a read exercises: those that changed its targets since they were last checked, less those that borrowed the read to learn their effect. */ +export const exercisedBy = ( + touched: TouchedTargets, + keys: readonly string[], + borrowers: ReadonlySet | undefined, +): Set => { + const exercised = new Set(); + for (const key of keys) + for (const name of touched.get(key)?.keys() ?? []) + if (!borrowers?.has(name)) exercised.add(name); + return exercised; +}; + +/** Record a modelled write on a target: the fields it wrote (undefined: all), when it was made and when it ended. */ +export const touchTarget = ( + touched: TouchedTargets, + pending: PendingTargets, + target: string, + write: { + readonly index: number; + readonly tool: string; + readonly end: number; + readonly fields: readonly string[] | undefined; + }, +): void => { + const held = pending.get(target); + pending.set(target, { + until: Math.max(held?.until ?? -Infinity, write.end), + at: (held?.at ?? new Map()).set(write.tool, write.index), + }); + const tools = touched.get(target) ?? new Map(); + const before = tools.get(write.tool); + tools.set( + write.tool, + write.fields === undefined || before === null + ? null + : [...new Set([...(before ?? []), ...write.fields])], + ); + touched.set(target, tools); +}; + +/** The fields of a write's effect on an entity no later write has overwritten: all of them for a create or delete (null). */ +type ChangedFields = readonly string[] | null; + +/** What a later write overwrites: the whole entity, the fields it sets, or every field (a recorded outcome). A document's fields are pointers, each covering the members under it. */ +interface Overwrite { + readonly whole: boolean; + readonly fields: ReadonlySet | "all"; + readonly pointers: boolean; +} + +/** Whether a write's fields cover a field: the same one, or, for a document's pointers, one above it. */ +const covers = (write: Overwrite, field: string): boolean => { + const { fields } = write; + if (fields === "all" || fields.has(field)) return true; + return ( + write.pointers && [...fields].some((over) => field.startsWith(`${over}/`)) + ); +}; + +/** What an earlier effect keeps after a write: nothing once every field it holds is overwritten; an update leaves a create's or delete's whole effect. */ +const remaining = ( + claim: ChangedFields, + write: Overwrite, +): ChangedFields | undefined => { + if (write.whole) return undefined; + if (claim === null) return null; + if (write.fields === "all") return undefined; + const left = claim.filter((field) => !covers(write, field)); + return left.length > 0 ? left : undefined; +}; + +/** Add a write's changed fields under a key: all of them once any write there changed the whole entity. */ +const record = ( + map: Map, + key: string, + whole: boolean, + changed: readonly string[], +): Map => { + const before = map.get(key); + return map.set( + key, + whole || before === null + ? null + : [...new Set([...(before ?? []), ...changed])], + ); +}; + +/** + * What each write changed since the targets it touched were last checked, less what later writes overwrote, so a + * write whose effect is gone drops out: from the entity's target, and from its collection's once no entity it changed + * there keeps any of it. + */ +export class WriteEffects { + /** Per entity target, the fields each tool changed there. */ + readonly #changed = new Map>(); + /** Per collection target, the entities each tool's claim there rests on, with the fields it changed on each. */ + readonly #members = new Map< + string, + Map> + >(); + + /** Forget a checked target's writes, as the touched targets do: all, or all but those a read showed nothing of. */ + checked(key: string, unseen?: ReadonlySet): void { + for (const byTool of [this.#changed, this.#members]) + if (unseen === undefined) byTool.delete(key); + else + for (const tool of byTool.get(key)?.keys() ?? []) + if (!unseen.has(tool)) byTool.get(key)!.delete(tool); + } + + /** Whether a read showing `fields` of a target shows anything of a tool's effect there: a create's or delete's always, an update's through a field it changed; unknown claims count as shown. */ + shows( + tool: string, + key: string, + fields: ReadonlySet | "all", + ): boolean { + const visible = (claim: ChangedFields | undefined): boolean => + fields === "all" || + claim === undefined || + claim === null || + claim.some((field) => fields.has(field)); + const own = this.#changed.get(key); + if (own !== undefined) return visible(own.get(tool)); + const members = this.#members.get(key)?.get(tool); + return members === undefined || [...members.values()].some(visible); + } + + /** A write's effect replaces what earlier writes left of theirs, then joins them. */ + supersede( + touched: TouchedTargets, + writer: string, + effect: { + readonly targets: readonly string[]; + readonly fields?: readonly string[]; + readonly changed?: { + readonly fields: readonly string[]; + readonly accumulated: readonly string[]; + }; + /** Set when the fields are a document's pointers. */ + readonly pointers?: boolean; + /** What the write overwrites of earlier effects, when more than the fields it claims. */ + readonly overwrites?: readonly string[]; + }, + ): void { + const [entity, collection] = effect.targets as [string, string?]; + const changed = effect.changed?.fields ?? effect.fields ?? []; + const whole = effect.fields === undefined; + this.#overwrite( + touched, + { entity, collection, writer }, + { + whole, + fields: new Set( + (effect.overwrites ?? changed).filter( + (field) => !effect.changed?.accumulated.includes(field), + ), + ), + pointers: effect.pointers === true, + }, + ); + this.#changed.set( + entity, + record(this.#changed.get(entity) ?? new Map(), writer, whole, changed), + ); + if (collection === undefined) return; + const byTool = this.#members.get(collection) ?? new Map(); + byTool.set( + writer, + record(byTool.get(writer) ?? new Map(), entity, whole, changed), + ); + this.#members.set(collection, byTool); + } + + /** A recorded outcome is the entity's new baseline: what earlier writes changed on it is gone, its membership is not. */ + observed(touched: TouchedTargets, targets: readonly string[]): void { + const [entity, collection] = targets as [string, string?]; + this.#overwrite( + touched, + { entity, collection, writer: undefined }, + { whole: false, fields: "all", pointers: false }, + ); + } + + #overwrite( + touched: TouchedTargets, + at: { + readonly entity: string; + readonly collection: string | undefined; + readonly writer: string | undefined; + }, + write: Overwrite, + ): void { + const own = this.#changed.get(at.entity); + for (const [tool, claim] of own ?? []) { + const left = tool === at.writer ? claim : remaining(claim, write); + if (left !== undefined) { + own!.set(tool, left); + if (tool !== at.writer) { + const tools = touched.get(at.entity); + const sent = tools?.get(tool); + // touched keeps sent members. A members check compares those; a surviving derived field stays on the effect. + if (tools !== undefined && sent !== undefined) + tools.set( + tool, + sent !== null && left !== null + ? sent.filter((field) => left.includes(field)) + : left, + ); + } + } else { + own!.delete(tool); + touched.get(at.entity)?.delete(tool); + } + } + if (at.collection === undefined) return; + const byTool = this.#members.get(at.collection); + for (const [tool, entities] of byTool ?? []) { + const claim = entities.get(at.entity); + if (tool === at.writer || claim === undefined) continue; + const left = remaining(claim, write); + if (left !== undefined) entities.set(at.entity, left); + else entities.delete(at.entity); + if (entities.size > 0) continue; + byTool!.delete(tool); + touched.get(at.collection)?.delete(tool); + } + } +} + +/** Of the tools a read exercised, those it shows something of on some target, and per target those it shows nothing of. */ +export const shownWrites = ( + touched: TouchedTargets, + effects: WriteEffects, + read: { + readonly keys: readonly string[]; + readonly exercised: ReadonlySet; + readonly fields: ReadonlySet | "all"; + }, +): { + readonly confirmed: string[]; + readonly unseen: Map>; +} => { + const confirmed: string[] = []; + const unseen = new Map>(); + for (const name of read.exercised) { + const on = read.keys.filter((key) => touched.get(key)?.has(name)); + const seen = on.filter((key) => effects.shows(name, key, read.fields)); + for (const key of on) + if (!seen.includes(key)) + unseen.set(key, (unseen.get(key) ?? new Set()).add(name)); + if (seen.length > 0) confirmed.push(name); + } + return { confirmed, unseen }; +}; + +/** A checked target forgets its writes, but for those a passing read showed nothing of, which a later check still takes. */ +export const forgetChecked = ( + pending: { + readonly touched: TouchedTargets; + readonly since: PendingTargets; + readonly effects: WriteEffects; + }, + key: string, + unseen: ReadonlySet | undefined, +): void => { + const writes = pending.touched.get(key); + if (unseen === undefined || writes === undefined) { + pending.touched.delete(key); + pending.effects.checked(key); + pending.since.delete(key); + return; + } + const at = pending.since.get(key)?.at; + for (const name of writes.keys()) + if (!unseen.has(name)) { + writes.delete(name); + at?.delete(name); + } + pending.effects.checked(key, unseen); +}; diff --git a/src/domain/analysisView/environmentStateWriteChecks.ts b/src/domain/analysisView/environmentStateWriteChecks.ts new file mode 100644 index 0000000000..590397910a --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteChecks.ts @@ -0,0 +1,585 @@ +import type { JsonValue } from "../jsonValue.js"; +import { iterateUrlQueryParameters } from "../recordedTransportUrl.js"; +import { + compareUnicodeCodePoints as compare, + compositeKey, +} from "../unicodeCodePointOrder.js"; +import { + at, + decodedSegment, + isObject, + isParam, + listItems, + missingRequiredFields, + renderStateEntity, + renderStateList, + replaceAt, + same, + scalarText, + segmentsOf, + type JsonObject, + type StateDocument, + type StateEntity, + type StateItemView, + type StateListView, + type StateRecordFacts, + type StateToolFacts, + type StateView, +} from "./environmentStateRules.js"; +import { pageBound, pagesOnly } from "./environmentStatePaging.js"; + +/** A query as replay compares a call's: decoded values by name, in any order; undefined when it repeats a name with another value, which no call's arguments can carry. */ +const queryKey = (query: string | undefined): string | undefined => { + const values = new Map(); + for (const [name, value] of iterateUrlQueryParameters(query ?? "")) { + if (values.has(name) && values.get(name) !== value) return undefined; + values.set(name, value); + } + return compositeKey( + [...values].sort(([left], [right]) => compare(left, right)), + ); +}; + +interface CheckedLine { + readonly ordinal: number; + readonly segments: readonly string[]; +} + +/** A list or document view's checked lines, by query, in the tool's recorded order, and which path segments are the tool's parameters. */ +export interface CheckedLines { + readonly params: ReadonlySet; + readonly byQuery: ReadonlyMap; + /** A whole list's lines that only page it, which serving answers a call supplying no query from. */ + readonly unfiltered: readonly CheckedLine[]; +} + +/** The line a read's call is answered from, and the collection serving lists for it. */ +export interface ServedLine { + readonly ordinal: number; + readonly collection: string; +} + +/** Each list or document view's checked lines. */ +export const checkedLines = (input: { + readonly tools: readonly StateToolFacts[]; + readonly records: readonly StateRecordFacts[]; + readonly views: ReadonlyMap; +}): Map => { + const byOrdinal = new Map( + input.records.map((record) => [record.ordinal, record]), + ); + const tools = new Map(input.tools.map((tool) => [tool.name, tool])); + const byView = new Map(); + for (const view of input.views.values()) { + const tool = tools.get(view.tool); + if (view.shape === "recorded" || view.shape === "item" || !tool) continue; + const checked = new Set(view.checked_record_ordinals); + const byQuery = new Map(); + const unfiltered: CheckedLine[] = []; + const whole = view.shape === "list" && view.membership === "all"; + for (const ordinal of tool.record_ordinals) { + const record = byOrdinal.get(ordinal); + const key = record === undefined ? undefined : queryKey(record.query); + if (!checked.has(ordinal) || key === undefined) continue; + const line = { ordinal, segments: segmentsOf(record!.path) }; + byQuery.set(key, [...(byQuery.get(key) ?? []), line]); + if (whole && pagesOnly(record!.query)) unfiltered.push(line); + } + const params = new Set( + segmentsOf(tool.path_template).flatMap((segment, index) => + isParam(segment) ? [index] : [], + ), + ); + byView.set(view.tool, { params, byQuery, unfiltered }); + } + return byView; +}; + +/** + * The checked line serving answers a read's call from, ranked as replay ranks lines: every + * parameter the read's value verbatim or once percent-decoded and the query the read's, then the + * most verbatim parameters, then recorded order. A read with no query falls back, as serving does, + * on a whole list's lines that only page it. + */ +export const servedLine = ( + lines: CheckedLines | undefined, + record: StateRecordFacts, +): ServedLine | undefined => { + const key = queryKey(record.query); + if (lines === undefined || key === undefined) return undefined; + return ( + bestLine(lines, lines.byQuery.get(key) ?? [], record) ?? + ([...iterateUrlQueryParameters(record.query ?? "")].length === 0 + ? bestLine(lines, lines.unfiltered, record) + : undefined) + ); +}; + +const bestLine = ( + lines: CheckedLines, + candidates: readonly CheckedLine[], + record: StateRecordFacts, +): ServedLine | undefined => { + const read = segmentsOf(record.path); + let best: { line: ServedLine; verbatim: number } | undefined; + for (const { ordinal, segments } of candidates) { + if (segments.length !== read.length) continue; + let verbatim = 0; + const matches = segments.every((segment, index) => { + if (segment === read[index]) { + if (lines.params.has(index)) verbatim += 1; + return true; + } + return lines.params.has(index) && decodedSegment(segment) === read[index]; + }); + if (matches && (best === undefined || verbatim > best.verbatim)) + best = { + line: { ordinal, collection: `/${segments.join("/")}` }, + verbatim, + }; + } + return best?.line; +}; + +export const entityTarget = (type: string, collection: string, id: string) => + `e${compositeKey([type, collection, id])}`; +export const collectionTarget = (type: string, collection: string) => + `c${compositeKey([type, collection])}`; +export const documentTarget = (tool: string) => `d${compositeKey([tool])}`; + +/** A target `remove` dropped, so a later absence can still name the collection and id serving resolved. */ +interface RemovedTarget { + readonly type: string; + readonly collection: string; + readonly id: string; +} + +/** The gate's copy of the state: entities in collection order and documents, changed by modelled writes. */ +export class SimulatedState { + readonly #entities = new Map(); + readonly #collections = new Map(); + readonly #documents = new Map(); + readonly #removed: RemovedTarget[] = []; + + constructor( + entities: readonly StateEntity[], + documents: readonly StateDocument[], + ) { + for (const entity of entities) this.put(entity); + for (const document of documents) + this.#documents.set(document.tool, document.value); + } + + entity( + type: string, + collection: string, + id: string, + ): StateEntity | undefined { + return this.#entities.get(entityTarget(type, collection, id)); + } + + members(type: string, collection: string): StateEntity[] { + return ( + this.#collections.get(collectionTarget(type, collection)) ?? [] + ).map((key) => this.#entities.get(key)!); + } + + ofType(type: string): StateEntity[] { + return [...this.#entities.values()].filter( + (entity) => entity.type === type, + ); + } + + /** Replace an entity in place, or add it at `index` of its collection (the end when omitted). */ + put(entity: StateEntity, index?: number): void { + const key = entityTarget(entity.type, entity.collection, entity.id); + const known = this.#entities.has(key); + this.#entities.set(key, entity); + if (known) return; + const collection = collectionTarget(entity.type, entity.collection); + const order = this.#collections.get(collection) ?? []; + order.splice(index ?? order.length, 0, key); + this.#collections.set(collection, order); + } + + remove(entity: StateEntity): void { + const key = entityTarget(entity.type, entity.collection, entity.id); + this.#entities.delete(key); + const collection = collectionTarget(entity.type, entity.collection); + const order = this.#collections.get(collection) ?? []; + this.#collections.set( + collection, + order.filter((member) => member !== key), + ); + const index = this.#removed.findIndex( + (target) => + target.type === entity.type && + target.collection === entity.collection && + target.id === entity.id, + ); + if (index >= 0) this.#removed.splice(index, 1); + this.#removed.push({ + type: entity.type, + collection: entity.collection, + id: entity.id, + }); + } + + /** Removed targets of this type the state does not hold again. */ + removedTargets(type: string): readonly RemovedTarget[] { + return this.#removed.filter( + (target) => + target.type === type && + !this.#entities.has(entityTarget(type, target.collection, target.id)), + ); + } + + document(tool: string): JsonValue | undefined { + return this.#documents.get(tool); + } + + setDocument(tool: string, value: JsonValue): void { + this.#documents.set(tool, value); + } +} + +/** An item read status that says the entity does not exist: 400, 404 or 410, as applications answer a missing id. */ +export const isAbsence = (status: number | null): boolean => + status === 400 || status === 404 || status === 410; + +/** A 2xx record: the only status whose response shows what a request did. */ +export const succeeded = (record: StateRecordFacts): boolean => + record.status !== null && record.status >= 200 && record.status < 300; + +/** Hand a base recording's body and a read's body to `use` together, parsing a recording once when both are the same. */ +export const withBodies = ( + withBody: (ordinal: number, use: (body: JsonValue | undefined) => U) => U, + ordinals: { readonly base: number; readonly read: number }, + use: (base: JsonValue | undefined, body: JsonValue | undefined) => T, +): T => { + const { base, read } = ordinals; + return base === read + ? withBody(read, (body) => use(body, body)) + : withBody(base, (served) => withBody(read, (body) => use(served, body))); +}; + +/** A check's outcome and the targets it covered. */ +export interface CheckOutcome { + readonly pass: boolean; + readonly targets: readonly string[]; + /** Entities the recording, or the served base of a recorded-membership list, lists that the state does not hold, by target. */ + readonly unheld: readonly string[]; + /** For a failed list read that differs only in some members' fields, those members' targets. */ + readonly differing?: readonly string[]; +} + +/** What a check compares: the read's recorded body and the served base recording it would answer on. */ +interface ReadBodies { + readonly body: JsonValue | undefined; + /** The recording whose envelope the served answer keeps: an item view's envelope, a list line's latest init recording. */ + readonly base: JsonValue | undefined; + readonly volatile: ReadonlySet; +} + +/** An item read: the answer the held entity renders into the served envelope must equal the recording; a recorded 404 or 410 agrees with none held. */ +/** How many of a held collection's segments match a path's verbatim, or undefined when one matches neither verbatim nor once percent-decoded. */ +const segmentMatch = ( + held: readonly string[], + read: readonly string[], +): number | undefined => { + if (held.length !== read.length) return undefined; + let verbatim = 0; + for (const [index, segment] of held.entries()) + if (segment === read[index]) verbatim += 1; + else if (decodedSegment(segment) !== read[index]) return undefined; + return verbatim; +}; + +/** A held target an item path names, preferring the path's own collection, then a verbatim spelling. */ +const matchingTarget = ( + path: { + readonly read: readonly string[]; + readonly ids: readonly string[]; + readonly written: string; + }, + collections: readonly string[], + held: (collection: string, id: string) => boolean, +): { readonly collection: string; readonly id: string } | undefined => { + for (const id of path.ids) + if (held(path.written, id)) return { collection: path.written, id }; + const ranked = [...new Set(collections)] + .flatMap((collection) => { + const verbatim = segmentMatch(segmentsOf(collection), path.read); + return verbatim === undefined ? [] : [{ collection, verbatim }]; + }) + .sort((left, right) => right.verbatim - left.verbatim); + for (const { collection } of ranked) + for (const id of path.ids) + if (held(collection, id)) return { collection, id }; + return undefined; +}; + +/** + * The entity an item read or write names, as serving names it: in a held collection its path matches, verbatim + * spellings first, the id percent-decoded once, then as written; else a removed target still named that way, so + * an absence keeps the collection and id a delete resolved; else the collection and decoded id as written. + */ +export const itemTarget = ( + state: SimulatedState, + type: string, + record: Pick, +): { readonly collection: string; readonly id: string } => { + const read = segmentsOf(record.path); + const raw = read.pop() ?? ""; + const path = { + read, + ids: [...new Set([decodedSegment(raw), raw])], + written: `/${read.join("/")}`, + }; + const removed = state.removedTargets(type); + return ( + matchingTarget( + path, + state.ofType(type).map((entity) => entity.collection), + (collection, id) => state.entity(type, collection, id) !== undefined, + ) ?? + matchingTarget( + path, + removed.map((target) => target.collection), + (collection, id) => + removed.some( + (target) => target.collection === collection && target.id === id, + ), + ) ?? { collection: path.written, id: path.ids[0]! } + ); +}; + +/** The held collection of a type a path names, as serving picks a create's: the most verbatim spelling, else the path as written. */ +export const heldCollection = ( + state: SimulatedState, + type: string, + path: string, +): string => { + const read = segmentsOf(path); + let held: string | undefined; + let verbatim = -1; + for (const collection of new Set( + state.ofType(type).map((entity) => entity.collection), + )) { + const matched = segmentMatch(segmentsOf(collection), read); + if (matched !== undefined && matched > verbatim) { + held = collection; + verbatim = matched; + } + } + return held ?? `/${read.join("/")}`; +}; + +export const checkItemRead = ( + state: SimulatedState, + view: StateItemView, + record: StateRecordFacts, + check: ReadBodies, +): CheckOutcome | undefined => { + const { collection, id } = itemTarget(state, view.type, record); + const entity = state.entity(view.type, collection, id); + const targets = [entityTarget(view.type, collection, id)]; + if (isAbsence(record.status) || (succeeded(record) && check.body === null)) + return { pass: entity === undefined, targets, unheld: [] }; + if (!succeeded(record) || check.base === undefined) return undefined; + const object = + check.body === undefined ? undefined : at(check.body, view.pointer); + // A usable body that no longer holds the entity is not what serving answers for one it holds. + if (!isObject(object)) + return check.body === undefined || entity === undefined + ? undefined + : { pass: false, targets, unheld: [] }; + if (entity === undefined) return { pass: false, targets, unheld: targets }; + // Serving refuses an entity lacking a field every recording carried, so no read can confirm one. + if (missingRequiredFields(entity, view, check.volatile).length > 0) + return { pass: false, targets, unheld: [] }; + const shown = renderStateEntity(entity, view, check.volatile, object); + return { + pass: same(replaceAt(check.base, view.pointer, shown), check.body), + targets, + unheld: [], + }; +}; + +/** A list read's bodies with the listed items of the read and of its served base. */ +interface ListBodies { + readonly body: JsonValue; + readonly base: JsonValue; + readonly items: readonly JsonObject[]; + readonly baseItems: readonly JsonObject[]; + readonly idField: string; + readonly volatile: ReadonlySet; + /** The collection serving lists: the answering line's. */ + readonly collection: string; +} + +/** The served list rendered from the base with `members`, compared with the read; volatile fields take the read's values. */ +const listCheck = ( + view: StateListView, + check: ListBodies, + members: readonly StateEntity[], +): Pick => { + if ( + members.some( + (entity) => + missingRequiredFields(entity, view, check.volatile).length > 0, + ) + ) + return { pass: false }; + const observed = new Map( + check.items.map((item) => [scalarText(item[check.idField]) ?? "", item]), + ); + const rendered = members.map((entity) => + renderStateEntity(entity, view, check.volatile, observed.get(entity.id)), + ); + if (same(renderStateList(view, check.base, rendered), check.body)) + return { pass: true }; + // When the recorded members alone account for the difference, only they differ. + const differs = members.filter((entity, index) => { + const item = observed.get(entity.id); + return item !== undefined && !same(rendered[index], item); + }); + const patched = rendered.map( + (item, index) => observed.get(members[index]!.id) ?? item, + ); + return differs.length > 0 && + same(renderStateList(view, check.base, patched), check.body) + ? { + pass: false, + differing: differs.map((entity) => + entityTarget(view.type, check.collection, entity.id), + ), + } + : { pass: false }; +}; + +/** A list read of every entity of a collection: membership, order, fields and counters must render from the state. */ +const checkAllList = ( + state: SimulatedState, + view: StateListView, + check: ListBodies, +): CheckOutcome => { + const { collection } = check; + const members = state.members(view.type, collection); + const held = new Set(members.map((entity) => entity.id)); + return { + ...listCheck(view, check, members), + targets: [ + collectionTarget(view.type, collection), + ...members.map((entity) => + entityTarget(view.type, collection, entity.id), + ), + ], + unheld: check.items + .map((item) => scalarText(item[check.idField]) ?? "") + .filter((id) => !held.has(id)) + .map((id) => entityTarget(view.type, collection, id)), + }; +}; + +/** A list read of recorded members: the served list holds the base's members the state still holds, so a member it dropped or never listed fails. */ +const checkRecordedList = ( + state: SimulatedState, + view: StateListView, + check: ListBodies, +): CheckOutcome => { + const { collection } = check; + const idOf = (item: JsonObject) => scalarText(item[check.idField]) ?? ""; + const members = check.baseItems.flatMap((item) => { + const entity = state.entity(view.type, collection, idOf(item)); + return entity === undefined ? [] : [entity]; + }); + const targets: string[] = []; + const unheld: string[] = []; + for (const id of new Set([...check.baseItems, ...check.items].map(idOf))) + (state.entity(view.type, collection, id) === undefined + ? unheld + : targets + ).push(entityTarget(view.type, collection, id)); + return { ...listCheck(view, check, members), targets, unheld }; +}; + +/** A list read on a checked line, compared with the answer the served environment renders on that line's base recording from the collection it names. */ +export const checkListRead = ( + state: SimulatedState, + view: StateListView, + record: StateRecordFacts, + check: ReadBodies & { readonly idField: string; readonly collection: string }, +): CheckOutcome | undefined => { + if (!succeeded(record)) return undefined; + const { body, base } = check; + const baseItems = + base === undefined ? undefined : listItems(base, view.pointer); + if (body === undefined || base === undefined || baseItems === undefined) + return undefined; + // Serving answers a page too small for the collection from its recording, so such a read checks nothing. + const size = pageBound(record.query, baseItems.length); + if ( + view.membership === "all" && + size !== undefined && + state.members(view.type, check.collection).length > size + ) + return undefined; + // A usable body that no longer holds the list is not what serving answers on this line. + const items = listItems(body, view.pointer); + const bodies = { ...check, body, base, items: items ?? [], baseItems }; + const outcome = + view.membership === "all" + ? checkAllList(state, view, bodies) + : checkRecordedList(state, view, bodies); + return items === undefined ? { ...outcome, pass: false } : outcome; +}; + +/** A document read on its checked line must equal the held document. */ +export const checkDocumentRead = ( + state: SimulatedState, + tool: string, + record: StateRecordFacts, + body: JsonValue | undefined, +): CheckOutcome | undefined => { + if (!succeeded(record)) return undefined; + const value = state.document(tool); + if (body === undefined || value === undefined) return undefined; + return { + pass: same(value, body), + targets: [documentTarget(tool)], + unheld: [], + }; +}; + +/** + * Check, field by field, the writes that changed an entity an unmodelled write also changed: + * each write's own fields must show as the state holds them, but for volatile ones, which take + * the read's value and so show nothing of a write. A create wrote every field, so it cannot be + * told apart from the unmodelled change and is not checked. + */ +export const writtenFieldChecks = ( + entity: StateEntity, + view: StateItemView, + observed: JsonObject, + check: { + readonly volatile: ReadonlySet; + readonly writes: ReadonlyMap; + }, +): Map => { + const shown = renderStateEntity(entity, view, check.volatile, observed); + const outcomes = new Map(); + for (const [tool, fields] of check.writes) { + const compared = (fields ?? []).filter( + (field) => + !check.volatile.has(field) && + (Object.hasOwn(observed, field) || Object.hasOwn(shown, field)), + ); + if (fields === null || compared.length === 0) continue; + outcomes.set( + tool, + compared.every((field) => same(shown[field], observed[field])), + ); + } + return outcomes; +}; diff --git a/src/domain/analysisView/environmentStateWriteCreate.ts b/src/domain/analysisView/environmentStateWriteCreate.ts new file mode 100644 index 0000000000..420a1ad18e --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteCreate.ts @@ -0,0 +1,333 @@ +import type { JsonValue } from "../jsonValue.js"; +import { ok } from "../result.js"; +import { + isObject, + isStateWrite, + scalarText, + type JsonObject, + type StateEntity, +} from "./environmentStateRules.js"; +import { createdFields } from "./environmentStateWriteApply.js"; +import { + collectionTarget, + entityTarget, + heldCollection, +} from "./environmentStateWriteChecks.js"; +import { + built, + callOf, + fail, + refuse, + requestObject, + retainFixed, + searchFrom, + startOf, + UNKNOWN, + type EffectContext, + type EffectResult, + type WriteDraft, + type WriteStep, +} from "./environmentStateWriteEffects.js"; +import { + explainCreate, + insertCandidates, + insertIndex, + nextInteger, + responseShapeOf, +} from "./environmentStateWriteExplain.js"; +import { + laterObservation, + newEntityIn, + type Observed, +} from "./environmentStateWriteObserve.js"; +import { + pathValuesOf, + type FieldRule, + type WriteCall, +} from "./environmentStateWriteRules.js"; +import { + integerOf, + isUuid, + nextSequence, + readClock, +} from "./environmentStateWriteValues.js"; + +/** The id a response names alone: a scalar body, or an object's id field or single scalar member. */ +const respondedId = ( + body: JsonValue | undefined, + idField: string, +): string | undefined => { + if (body === undefined) return undefined; + if (!isObject(body)) return scalarText(body); + if (Object.hasOwn(body, idField)) return scalarText(body[idField]); + const members = Object.values(body); + return members.length === 1 ? scalarText(members[0]) : undefined; +}; + +/** The written entity a create's response or a later read shows. */ +const createdObservation = ( + context: EffectContext, + step: WriteStep, + target: { + readonly type: string; + readonly collection: string; + readonly idField: string; + }, +): Observed | undefined => { + const held = (id: string) => + context.state.entity(target.type, target.collection, id) !== undefined; + const own = context.io.withBody(step.record.ordinal, (body) => + newEntityIn(step.record.ordinal, body, target.idField, held), + ); + if (own !== undefined) return own; + // A response that names the created id alone ties the later read to that id. + const id = context.io.withBody(step.record.ordinal, (body) => + respondedId(body, target.idField), + ); + const later = laterObservation(searchFrom(context, step), { + ...target, + ...(id === undefined ? {} : { id }), + held, + isWrite: isStateWrite, + }); + if (later !== undefined) context.observed(later.ordinal); + return later; +}; + +const fieldsOf = (entities: readonly StateEntity[]): JsonObject[] => + entities.map((entity) => entity.fields); + +/** Build a create plan from its exemplar's observation. */ +const buildCreate = ( + context: EffectContext, + draft: WriteDraft, + step: WriteStep, + evidence: { + readonly observed: Observed; + readonly call: WriteCall & { readonly body: JsonObject }; + readonly collection: string; + }, +): void => { + const type = draft.plan.target!; + const idField = context.idField(type); + const members = fieldsOf(context.state.members(type, evidence.collection)); + const explained = explainCreate({ + entity: evidence.observed.object, + call: evidence.call, + idField, + startMs: startOf(context, step), + collection: members, + type: fieldsOf(context.state.ofType(type)), + }); + const clocks = explained.rules.flatMap((rule) => + rule.rule === "clock" ? [rule.field] : [], + ); + // Identity and clock orders first, then any other scalar field the collection may be sorted by. + const scalars = Object.keys(members[0] ?? {}).filter((field) => + members.every((member) => + ["string", "number"].includes(typeof member[field]), + ), + ); + draft.candidates = insertCandidates(members, [ + ...new Set([idField, ...clocks, ...scalars]), + ]); + retainFixed(context, explained.body.fixed); + Object.assign(draft.plan, { + exemplar_ordinal: step.record.ordinal, + observation: { + ordinal: evidence.observed.ordinal, + pointer: evidence.observed.pointer, + }, + response: context.io.withBody(step.record.ordinal, (body) => + responseShapeOf( + step.record.ordinal, + body, + context.io.emptyResponse(step.record.ordinal), + { idField, id: evidence.observed.id }, + ), + ), + body: explained.body, + rules: explained.rules, + collection: explained.tied ? evidence.collection : null, + insert: draft.candidates[0]!, + }); + draft.observed = evidence.observed.object; +}; + +type SequenceRule = Extract; + +/** The sequences a create's plan generates, each with the key serving counts it under. */ +const sequencesOf = ( + draft: WriteDraft, + collection: string, +): { readonly rule: SequenceRule; readonly key: string }[] => + draft.plan.rules.flatMap((rule) => + rule.rule === "sequence" + ? [ + { + rule, + key: + rule.scope === "collection" + ? `${rule.field} ${collection}` + : rule.field, + }, + ] + : [], + ); + +/** Whether a later call's sequences hold what serving generates: one past the largest value their scope holds or started with, and past the plan's last. */ +const followsSequences = ( + context: EffectContext, + draft: WriteDraft, + created: Pick, +): boolean => + sequencesOf(draft, created.collection).every(({ rule, key }) => { + const scope = [ + ...context.state.ofType(created.type), + ...context.startedWith(created.type), + ].filter( + (entity) => + rule.scope === "type" || entity.collection === created.collection, + ); + const next = nextSequence( + nextInteger(fieldsOf(scope), rule.field), + draft.sequences.get(key), + ); + return integerOf(created.fields[rule.field]) === next; + }); + +/** Whether a later call's generated values take the forms serving generates them in: clocks in the learned format, unique values and UUID identities as UUIDs, and sequences in order. */ +const followsGenerated = ( + context: EffectContext, + draft: WriteDraft, + created: Pick, +): boolean => + draft.plan.rules.every((rule) => { + const value = created.fields[rule.field]; + if (rule.rule === "clock") + return ( + JSON.stringify(readClock(value)?.format) === JSON.stringify(rule.format) + ); + if (rule.rule === "unique") + return ( + isUuid(value) && + !context.state + .ofType(created.type) + .some((held) => held.fields[rule.field] === value) + ); + return ( + rule.rule !== "identity" || rule.generator !== "uuid" || isUuid(value) + ); + }) && followsSequences(context, draft, created); + +/** Record the values a create's sequences took, which the next call follows. */ +const takeSequences = ( + draft: WriteDraft, + created: Pick, +): void => { + for (const { rule, key } of sequencesOf(draft, created.collection)) { + const value = integerOf(created.fields[rule.field]); + if (value !== undefined) draft.sequences.set(key, value); + } +}; + +/** Add a created entity to the state at the plan's insert position, or last before the plan has one. */ +const insertCreated = ( + context: EffectContext, + draft: WriteDraft, + entity: StateEntity, +): string => { + const members = fieldsOf( + context.state.members(entity.type, entity.collection), + ); + context.io.retain(64 + JSON.stringify(entity.fields).length * 2); + context.state.put( + entity, + draft.plan.insert === null + ? members.length + : insertIndex(draft.plan.insert, members, entity.fields), + ); + return entityTarget(entity.type, entity.collection, entity.id); +}; + +/** Apply a recorded create, with the values it recorded for generated fields; an unexplained create adds what it recorded. */ +export const applyCreate = ( + context: EffectContext, + draft: WriteDraft, + step: WriteStep, +): EffectResult => { + const type = draft.plan.target!; + const idField = context.idField(type); + // The held collection serving would pick for this path, as an item read's names its entity. + const collection = heldCollection(context.state, type, step.record.path); + const body = requestObject(context, step.record); + const observed = createdObservation(context, step, { + type, + collection, + idField, + }); + if (observed === undefined) { + fail(draft, step, body === undefined ? "body-unusable" : "no-observation"); + return UNKNOWN; + } + const entity = (fields: JsonObject, id: string): StateEntity => ({ + type, + collection, + id, + fields, + view_fields: [], + record_ordinals: [], + }); + const recorded = (): EffectResult => { + const inserted = insertCreated( + context, + draft, + entity(observed.object, observed.id), + ); + return { + kind: "observed", + targets: [inserted, collectionTarget(type, collection)], + entity: { type, collection, id: observed.id }, + }; + }; + if (step.observeOnly || body === undefined) { + fail(draft, step, "body-unusable"); + return recorded(); + } + const call = { ...callOf(step, body), body }; + // The exemplar defines the plan's sequences; each later call must follow them. + const later = built(draft); + if (!later) buildCreate(context, draft, step, { observed, call, collection }); + // Rules learn from the recorded path; serving fills path fields from the collection it selects. + const created = { + ...call, + pathValues: + pathValuesOf(step.tool.path_template, collection) ?? call.pathValues, + }; + const fields = createdFields(draft.plan, draft.observed!, created, (rule) => + ok(observed.object[rule.field] ?? null), + ); + const id = fields.ok ? scalarText(fields.value[idField]) : undefined; + if ( + !fields.ok || + id === undefined || + context.state.entity(type, collection, id) !== undefined || + (later && + !followsGenerated(context, draft, { + type, + collection, + fields: fields.value, + })) + ) { + refuse(draft, step, "unexplained-change"); + return recorded(); + } + takeSequences(draft, { collection, fields: fields.value }); + const inserted = insertCreated(context, draft, entity(fields.value, id)); + return { + kind: "applied", + targets: [inserted, collectionTarget(type, collection)], + entity: { type, collection, id }, + inserted, + }; +}; diff --git a/src/domain/analysisView/environmentStateWriteDocument.ts b/src/domain/analysisView/environmentStateWriteDocument.ts new file mode 100644 index 0000000000..886fd5ed6f --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteDocument.ts @@ -0,0 +1,160 @@ +import type { JsonValue } from "../jsonValue.js"; +import { + at as valueAt, + decodedSegment, + isObject, + pointerTo, + same, + segmentsOf, + type JsonObject, +} from "./environmentStateRules.js"; +import { + documentBodyProblem, + writtenDocument, +} from "./environmentStateWriteApply.js"; +import { documentTarget } from "./environmentStateWriteChecks.js"; +import { + built, + callOf, + fail, + refuse, + requestObject, + UNKNOWN, + type EffectContext, + type EffectResult, + type WriteDraft, + type WriteStep, +} from "./environmentStateWriteEffects.js"; +import { documentPointerOf } from "./environmentStateWriteExplain.js"; + +/** Whether a write's path names the held document's, a segment verbatim or once percent-decoded, as serving matches a call. */ +const namesDocument = (path: string, held: string): boolean => { + const written = segmentsOf(path); + const recorded = segmentsOf(held); + return ( + written.length === recorded.length && + recorded.every( + (segment, index) => + segment === written[index] || + decodedSegment(segment) === written[index], + ) + ); +}; + +/** Where a write goes: a built plan writes where serving does, at its own object; an exemplar at the object holding most of its members. */ +const writePointer = ( + draft: WriteDraft, + value: JsonValue, + body: JsonObject, +): string | undefined => + built(draft) + ? draft.plan.observation?.pointer + : documentPointerOf(value, body); + +/** + * The fields a document write changed, as pointers: a merge the members it sent at its object, a replacement every + * member it wrote or removed there, so later writes can overwrite its effect (undefined: the whole document). + */ +const writtenPointers = ( + draft: WriteDraft, + body: JsonObject, + values: { readonly before: JsonValue; readonly after: JsonValue }, +): readonly string[] | undefined => { + const at = draft.plan.observation?.pointer ?? ""; + if (draft.plan.merge === "merge") + return Object.keys(body).map((member) => `${at}${pointerTo(member)}`); + const members = new Set( + [values.before, values.after].flatMap((value) => { + const object = valueAt(value, at); + return isObject(object) ? Object.keys(object) : []; + }), + ); + if (members.size === 0) return at === "" ? undefined : [at]; + return [...members].map((member) => `${at}${pointerTo(member)}`); +}; + +/** Build a document plan from its exemplar: its object, merge or replacement by method, and the members it sent. */ +const buildDocument = ( + draft: WriteDraft, + step: WriteStep, + evidence: { readonly pointer: string; readonly body: JsonObject }, +): void => { + const { ordinal } = step.record; + Object.assign(draft.plan, { + exemplar_ordinal: ordinal, + observation: { ordinal, pointer: evidence.pointer }, + merge: step.tool.method === "PUT" ? "replace" : "merge", + body: { + accepted: Object.keys(evidence.body), + fixed: [], + required: [], + identity: null, + kinds: [], + }, + }); +}; + +/** Apply a recorded document write: the body merged into, or replacing, the object holding most of its members. */ +export const applyDocument = ( + context: EffectContext, + draft: WriteDraft, + step: WriteStep, +): EffectResult => { + const tool = draft.plan.target!; + const initial = context.initialDocument(tool); + // Serving refuses a write at another path, so one here changes no document the state holds. + if (initial !== undefined && !namesDocument(step.record.path, initial.path)) + return UNKNOWN; + const value = context.state.document(tool); + const body = requestObject(context, step.record); + const pointer = + value === undefined || body === undefined + ? undefined + : writePointer(draft, value, body); + if (step.observeOnly || pointer === undefined) { + fail(draft, step, body === undefined ? "body-unusable" : "no-observation"); + return UNKNOWN; + } + if (!built(draft)) buildDocument(draft, step, { pointer, body: body! }); + const call = callOf(step, body!); + // Serving refuses a body its check refuses, so such a call cannot confirm the plan. + if ( + documentBodyProblem(draft.plan, call, { + initial: initial?.value, + current: value!, + }) !== undefined + ) { + refuse(draft, step, "unexplained-change"); + return UNKNOWN; + } + const written = writtenDocument(draft.plan, value!, call); + if (!written.ok) { + refuse(draft, step, "no-observation"); + return UNKNOWN; + } + const { ordinal } = step.record; + draft.plan.response ??= context.io.withBody(ordinal, (response) => ({ + ordinal, + shape: context.io.emptyResponse(ordinal) + ? "empty" + : response !== undefined && same(response, written.value) + ? "entity" + : "recorded", + pointer: null, + })); + context.io.retain(64 + JSON.stringify(written.value).length * 2); + context.state.setDocument(tool, written.value); + const fields = writtenPointers(draft, body!, { + before: value!, + after: written.value, + }); + const replaced = draft.plan.merge === "merge" ? undefined : pointer; + return { + kind: "applied", + targets: [documentTarget(tool)], + ...(fields === undefined ? {} : { fields, pointers: true }), + ...(fields === undefined || replaced === undefined + ? {} + : { overwrites: [replaced] }), + }; +}; diff --git a/src/domain/analysisView/environmentStateWriteEffects.ts b/src/domain/analysisView/environmentStateWriteEffects.ts new file mode 100644 index 0000000000..dc34ab6bec --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteEffects.ts @@ -0,0 +1,559 @@ +import type { JsonValue } from "../jsonValue.js"; +import { ok } from "../result.js"; +import { + isObject, + isStateWrite, + locateItem, + at, + same, + type JsonObject, + type StateDocument, + type StateEntity, + type StateRecordFacts, + type StateToolFacts, + type StateType, + type StateView, +} from "./environmentStateRules.js"; +import { updatedEntity, updatedFields } from "./environmentStateWriteApply.js"; +import { + collectionTarget, + entityTarget, + itemTarget, + type SimulatedState, +} from "./environmentStateWriteChecks.js"; +import { + conflictsWithPlan, + explainUpdate, + refinedBody, + responseShapeOf, +} from "./environmentStateWriteExplain.js"; +import { + laterObservation, + shownBefore, + viewFields, + type Observed, +} from "./environmentStateWriteObserve.js"; +import { integerOf, readClock, slugOf } from "./environmentStateWriteValues.js"; +import { + pathValuesOf, + type FieldRule, + type InsertRule, + type StateWritePlan, + type WriteCall, + type WriteReason, +} from "./environmentStateWriteRules.js"; + +type Mutable = { -readonly [K in keyof T]: T[K] }; + +/** A plan under construction and the evidence the walk gathers for its verdict. */ +export interface WriteDraft { + readonly plan: Mutable; + /** The exemplar's observed entity, whose unruled fields a create keeps. */ + observed: JsonObject | undefined; + candidates: InsertRule[]; + readonly confirmed: number[]; + contradictedBy: number | undefined; + sideEffect: boolean; + calls: number; + /** The scalar values each modelled update call showed and sent, which later rules must reproduce. */ + readonly updates: { + readonly shown: ReadonlyMap; + readonly sent: ReadonlyMap; + }[]; + /** The last value each sequence took, by field and, for a collection's sequence, collection; the next call must follow it. */ + readonly sequences: Map; +} + +/** A write observation plus the fields its selected read view actually covers, if any. */ +type UpdateObservation = Observed & { + readonly readFields?: readonly string[]; +}; + +/** Response bodies, request bodies and budget the effects read. */ +export interface WriteIo { + readonly withBody: ( + ordinal: number, + use: (body: JsonValue | undefined) => T, + ) => T; + readonly requestBody: (ordinal: number) => JsonValue | undefined; + readonly emptyResponse: (ordinal: number) => boolean; + readonly retain: (bytes: number) => void; +} + +/** What the write gate reads: the manifest tools, the timeline, the read views and the initial state. */ +export interface WriteSimulationInput { + readonly tools: readonly StateToolFacts[]; + readonly records: readonly StateRecordFacts[]; + readonly timed: boolean; + readonly views: ReadonlyMap; + readonly types: readonly StateType[]; + readonly entities: readonly StateEntity[]; + readonly documents: readonly StateDocument[]; + readonly volatileOf: (type: string) => ReadonlySet; + readonly io: WriteIo; +} + +/** What applying a recorded write reads besides the draft and the call. */ +export interface EffectContext { + readonly state: SimulatedState; + readonly io: WriteIo; + readonly records: readonly StateRecordFacts[]; + readonly timed: boolean; + readonly viewOf: (ordinal: number) => StateView | undefined; + readonly idField: (type: string) => string; + /** Mark a later read as a create's or update's observation, so it does not check that write. */ + readonly observed: (ordinal: number) => void; + /** Whether an unmodelled write may have changed the entity since the state last saw it. */ + readonly unknown: (type: string, collection: string, id: string) => boolean; + /** A document as the initial state holds it: its path, and the value serving also reads member types from. */ + readonly initialDocument: (tool: string) => StateDocument | undefined; + /** The entities of a type the initial state held, whose values a sequence never reuses. */ + readonly startedWith: (type: string) => readonly StateEntity[]; +} + +/** One recorded write: the record, its timeline index, its tool, and whether its plan only takes the recorded outcome. */ +export interface WriteStep { + readonly record: StateRecordFacts; + readonly index: number; + readonly tool: StateToolFacts; + /** Set once the plan has a reason: the write only brings the state to its recorded outcome. */ + readonly observeOnly: boolean; +} + +/** + * What applying a recorded write did: the modelled effect (attributable to the tool), the + * recorded outcome when the model does not explain it (not attributable), or nothing known. + */ +export type EffectResult = + | { + readonly kind: "applied"; + readonly targets: readonly string[]; + readonly entity?: WrittenEntity; + /** The fields an update wrote, or the pointers a document write wrote; a create writes the whole entity. */ + readonly fields?: readonly string[]; + /** Set when the fields are a document's pointers, each covering the members under it. */ + readonly pointers?: boolean; + /** A document replacement's object, as a pointer: it overwrites every earlier effect under it, though it claims only its members. */ + readonly overwrites?: readonly string[]; + /** The fields an update sent or changed, derived ones included, and those of them its increments add to. */ + readonly changed?: { + readonly fields: readonly string[]; + readonly accumulated: readonly string[]; + }; + readonly inserted?: string; + } + | { + readonly kind: "observed"; + readonly targets: readonly string[]; + readonly entity?: WrittenEntity; + } + | { readonly kind: "unknown" }; + +/** The entity a write changed, by type, collection and id. */ +export interface WrittenEntity { + readonly type: string; + readonly collection: string; + readonly id: string; +} + +export const UNKNOWN: EffectResult = { kind: "unknown" }; + +export const built = (draft: WriteDraft): boolean => + draft.plan.exemplar_ordinal !== null; + +/** Give an unbuilt plan its reason; a built plan keeps its model. */ +export const fail = ( + draft: WriteDraft, + step: WriteStep, + reason: WriteReason, +): void => { + if (!built(draft) && !step.observeOnly) draft.plan.reason = reason; +}; + +/** A recorded call the built model does not explain disqualifies the plan. */ +export const refuse = ( + draft: WriteDraft, + step: WriteStep, + reason: WriteReason, +): void => { + if (!step.observeOnly) draft.plan.reason = reason; +}; + +export const requestObject = ( + context: EffectContext, + record: StateRecordFacts, +): JsonObject | undefined => { + const body = context.io.requestBody(record.ordinal); + return isObject(body) ? body : undefined; +}; + +export const callOf = ( + step: WriteStep, + body: JsonObject | undefined, +): WriteCall => ({ + path: step.record.path, + pathValues: + pathValuesOf(step.tool.path_template, step.record.path) ?? new Map(), + body, +}); + +export const startOf = ( + context: EffectContext, + step: WriteStep, +): number | undefined => (context.timed ? step.record.start : undefined); + +export const searchFrom = (context: EffectContext, step: WriteStep) => ({ + records: context.records, + from: step.index, + ...(context.timed ? { after: step.record.end } : {}), + viewOf: context.viewOf, + withBody: context.io.withBody, +}); + +/** The entity an update's response shows, located by its identity, else a later item read of the target the write resolved. */ +const updatedObservation = ( + context: EffectContext, + step: WriteStep, + entity: { + readonly type: string; + readonly idField: string; + readonly collection: string; + readonly id: string; + }, + required: readonly string[], +): UpdateObservation | undefined => { + const own = context.io.withBody( + step.record.ordinal, + (body): Observed | undefined => { + const located = + body === undefined ? undefined : locateItem(body, entity.id); + const object = + located === undefined ? undefined : at(body!, located.pointer); + return located?.keys.includes(entity.idField) === true && isObject(object) + ? { + ordinal: step.record.ordinal, + pointer: located.pointer, + object, + id: entity.id, + } + : undefined; + }, + ); + if ( + own !== undefined && + required.every((field) => Object.hasOwn(own.object, field)) + ) + return own; + const later = laterObservation(searchFrom(context, step), { + type: entity.type, + collection: entity.collection, + idField: entity.idField, + id: entity.id, + held: () => true, + isWrite: isStateWrite, + sameItem: (record) => { + const found = itemTarget(context.state, entity.type, record); + return found.collection === entity.collection && found.id === entity.id; + }, + // With an answer or without one, only a read whose view shows every learned clock shows what this call stamped. + covers: (view) => required.every((field) => view.fields.includes(field)), + }); + if (later === undefined) return own; + const readFields = viewFields(context.viewOf(later.ordinal)); + context.observed(later.ordinal); + const object = { ...own?.object, ...later.object }; + // For covered fields, the later read's omission is evidence too; do not fill it back + // from an earlier write response. Fields outside that view retain response evidence. + for (const field of readFields ?? []) + if (!Object.hasOwn(later.object, field)) delete object[field]; + return { + ...later, + object, + ...(readFields === undefined ? {} : { readFields }), + }; +}; + +/** Add what a later call of an update showed to its plan: members, their stored types, fixed values and rules not seen before. */ +const refineUpdate = ( + context: EffectContext, + draft: WriteDraft, + evidence: { + readonly after: Observed; + readonly explained: ReturnType; + }, +): void => { + const { plan } = draft; + const body = refinedBody(plan.body!, evidence.explained.body, { + observed: draft.observed!, + stored: evidence.after.object, + }); + retainFixed(context, body.fixed.slice(plan.body!.fixed.length)); + plan.body = body; + const ruled = new Set(plan.rules.map((rule) => rule.field)); + plan.rules = [ + ...plan.rules, + ...evidence.explained.rules.filter((rule) => !ruled.has(rule.field)), + ]; +}; + +/** Count the sent values of fixed members a plan keeps, which outlive the request body they were read from. */ +export const retainFixed = ( + context: EffectContext, + fixed: readonly { readonly sent: JsonValue }[], +): void => { + if (fixed.length > 0) + context.io.retain(64 + JSON.stringify(fixed).length * 2); +}; + +/** Whether a rule changes its field on calls that do not send it: serving applies it to every call that could. */ +const unsentChange = (rule: FieldRule): boolean => + rule.rule === "increment" || rule.rule === "slug"; + +/** A call's scalar members by name, as much of it as later rules are checked against. */ +const scalars = (object: JsonObject): Map => + new Map( + Object.entries(object).filter( + ([, value]) => value === null || typeof value !== "object", + ), + ); + +/** Whether a rule a later call showed gives an earlier call a value other than the one it showed for a field it did not send. */ +const contradictsEarlierCalls = ( + draft: WriteDraft, + rules: readonly FieldRule[], +): boolean => { + const ruled = new Set(draft.plan.rules.map((rule) => rule.field)); + return rules.some( + (rule) => + (unsentChange(rule) || rule.rule === "clock") && + !ruled.has(rule.field) && + draft.updates.some(({ shown, sent }) => { + // An unruled field an earlier call showed is one it left as it was. + const value = shown.get(rule.field); + if (value === undefined) return false; + if (rule.rule === "increment") return integerOf(value) !== undefined; + // Serving stamps a clock on every call, so a field an earlier call kept contradicts it. + if (rule.rule === "clock") return true; + const from = rule.rule === "slug" ? sent.get(rule.from) : undefined; + return typeof from === "string" && slugOf(from) !== value; + }), + ); +}; + +/** Build an update plan from its exemplar, or add what a later call showed. */ +const buildUpdate = ( + context: EffectContext, + draft: WriteDraft, + step: WriteStep, + evidence: { + readonly after: Observed; + readonly explained: ReturnType; + readonly idField: string; + }, +): void => { + if (built(draft)) { + refineUpdate(context, draft, evidence); + return; + } + retainFixed(context, evidence.explained.body.fixed); + const { ordinal } = step.record; + Object.assign(draft.plan, { + exemplar_ordinal: ordinal, + observation: { + ordinal: evidence.after.ordinal, + pointer: evidence.after.pointer, + }, + response: context.io.withBody(ordinal, (response) => + responseShapeOf(ordinal, response, context.io.emptyResponse(ordinal), { + idField: evidence.idField, + id: evidence.after.id, + }), + ), + body: evidence.explained.body, + rules: evidence.explained.rules, + }); + draft.observed = evidence.after.object; +}; + +/** + * Apply a recorded update to the held entity, explaining every change it made. An update the + * model does not explain, or of an entity an unmodelled write may have changed, brings the + * entity to what it recorded instead and is not attributed to the tool. + */ +export const applyUpdate = ( + context: EffectContext, + draft: WriteDraft, + step: WriteStep, +): EffectResult => { + const type = draft.plan.target!; + const idField = context.idField(type); + // The entity as serving names it. + const { collection, id } = itemTarget(context.state, type, step.record); + const entity = context.state.entity(type, collection, id); + const after = + entity === undefined + ? undefined + : updatedObservation( + context, + step, + { type, idField, collection, id }, + draft.plan.rules.flatMap((rule) => + rule.rule === "clock" ? [rule.field] : [], + ), + ); + if (entity === undefined || after === undefined) { + fail(draft, step, "no-observation"); + return UNKNOWN; + } + const targets = [ + entityTarget(type, collection, id), + collectionTarget(type, collection), + ]; + const target = { type, collection, id }; + const recorded = (): EffectResult => { + context.state.put( + updatedEntity(entity, { ...entity.fields, ...after.object }), + ); + return { kind: "observed", targets, entity: target }; + }; + const body = requestObject(context, step.record); + if (step.observeOnly || body === undefined) { + fail(draft, step, "body-unusable"); + return recorded(); + } + const unknown = context.unknown(type, collection, id); + const call = { ...callOf(step, body), body }; + const explained = explainUpdate({ + before: entity.fields, + after: after.object, + call, + method: step.tool.method, + idField, + id, + startMs: startOf(context, step), + ...(after.readFields === undefined ? {} : { readFields: after.readFields }), + shown: shownBefore(context.viewOf, entity, after.ordinal), + }); + if (explained.unexplained.length > 0 && !unknown) { + refuse(draft, step, "unexplained-change"); + return recorded(); + } + // Each call must agree with the plan; one on an unknown entity cannot be checked, so it adds nothing to the plan. + if ( + built(draft) && + (unknown || contradictsPlan(draft, entity.fields, after.object, explained)) + ) { + if (!unknown) refuse(draft, step, "unexplained-change"); + return recorded(); + } + buildUpdate(context, draft, step, { after, explained, idField }); + // Serving reads the plan's observation, so the gate does too; generated values are this call's. + const written = updatedFields( + draft.plan, + { before: entity.fields, observed: draft.observed! }, + { ...call, id }, + (rule) => ok(after.object[rule.field]), + ); + const disagrees = + written.ok && + draft.plan.rules.some( + (rule) => + unsentChange(rule) && + Object.hasOwn(after.object, rule.field) && + !same(written.value[rule.field], after.object[rule.field]), + ); + if (!written.ok || (disagrees && !unknown)) + refuse(draft, step, "unexplained-change"); + if (!written.ok || disagrees || unknown) return recorded(); + const facts = { shown: scalars(after.object), sent: scalars(body) }; + context.io.retain( + 64 + JSON.stringify([...facts.shown, ...facts.sent]).length * 2, + ); + draft.updates.push(facts); + context.io.retain(64 + JSON.stringify(written.value).length * 2); + context.state.put(updatedEntity(entity, written.value)); + // A later read that also shows an unmodelled change checks only the members this call sent. + const sent = Object.keys(body).filter( + (member) => member !== draft.plan.body?.identity, + ); + const derived = Object.keys(written.value).filter( + (field) => !same(written.value[field], entity.fields[field]), + ); + const changed = { + fields: [...new Set([...sent, ...derived])], + accumulated: draft.plan.rules.flatMap((rule) => + rule.rule === "increment" ? [rule.field] : [], + ), + }; + return { kind: "applied", targets, entity: target, fields: sent, changed }; +}; + +/** Serving regenerates a learned clock on every update, so recorded calls must advance it. */ +const contradictsGeneratedClock = ( + before: JsonObject, + after: JsonObject, + rules: readonly FieldRule[], + explained: readonly FieldRule[], +): boolean => + rules.some((rule) => { + if (rule.rule !== "clock" || !Object.hasOwn(after, rule.field)) + return false; + const previous = readClock(before[rule.field]); + if (previous === undefined) return false; + const next = readClock(after[rule.field]); + return ( + !explained.some( + (candidate) => + candidate.field === rule.field && candidate.rule === "clock", + ) || + next === undefined || + next.ms <= previous.ms + ); + }); + +/** Whether a later call explains its changes otherwise than the plan: rules for unsent changes hold for every call, so each call must agree with the others. */ +const contradictsPlan = ( + draft: WriteDraft, + before: JsonObject, + after: JsonObject, + explained: ReturnType, +): boolean => + contradictsEarlierCalls(draft, explained.rules) || + contradictsGeneratedClock(before, after, draft.plan.rules, explained.rules) || + conflictsWithPlan(draft.plan, explained); + +/** Apply a recorded delete: the held entity leaves its collection. */ +export const applyDelete = ( + context: EffectContext, + draft: WriteDraft, + step: WriteStep, +): EffectResult => { + const type = draft.plan.target!; + const { collection, id } = itemTarget(context.state, type, step.record); + const entity = context.state.entity(type, collection, id); + if (entity === undefined) { + fail(draft, step, "no-observation"); + return UNKNOWN; + } + const { ordinal } = step.record; + if (!built(draft) && !step.observeOnly) + Object.assign(draft.plan, { + exemplar_ordinal: ordinal, + response: context.io.withBody(ordinal, (body) => + responseShapeOf(ordinal, body, context.io.emptyResponse(ordinal), { + idField: context.idField(type), + id, + }), + ), + }); + context.state.remove(entity); + const targets = [ + entityTarget(type, collection, id), + collectionTarget(type, collection), + ]; + return { + kind: step.observeOnly ? "observed" : "applied", + targets, + entity: { type, collection, id }, + }; +}; diff --git a/src/domain/analysisView/environmentStateWriteExplain.test.ts b/src/domain/analysisView/environmentStateWriteExplain.test.ts new file mode 100644 index 0000000000..0ab067888b --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteExplain.test.ts @@ -0,0 +1,643 @@ +import { expect, it } from "vitest"; +import type { JsonValue } from "../jsonValue.js"; +import type { JsonObject } from "./environmentStateRules.js"; +import { + compareValues, + documentPointerOf, + explainCreate, + explainUpdate, + insertCandidates, + insertIndex, + responseShapeOf, + type CreateEvidence, + type UpdateEvidence, +} from "./environmentStateWriteExplain.js"; +import { + classifyWrite, + pathValuesOf, + prefixRelated, + type FieldRule, + type InsertRule, + type WriteEffect, + type WriteResponse, +} from "./environmentStateWriteRules.js"; + +const START = Date.UTC(2026, 2, 4, 5, 6, 0); +const MINUTE = 60_000; +const UUID = "123e4567-e89b-42d3-a456-426614174000"; +const OTHER_UUID = "00000000-0000-4000-8000-000000000000"; + +const created = ( + entity: JsonObject, + body: JsonObject, + context: Partial & { + readonly path?: Record; + } = {}, +): CreateEvidence => { + const { path = {}, ...rest } = context; + return { + entity, + call: { path: "/items", pathValues: new Map(Object.entries(path)), body }, + idField: "id", + startMs: START, + collection: [], + type: [], + ...rest, + }; +}; + +const updated = ( + before: JsonObject, + after: JsonObject, + body: JsonObject, + context: Partial = {}, +): UpdateEvidence => ({ + before, + after, + call: { path: "/items/42", pathValues: new Map([["item", "42"]]), body }, + method: "PATCH", + idField: "id", + id: "42", + startMs: START, + ...context, +}); + +const ruleFor = ( + rules: readonly FieldRule[], + field: string, +): FieldRule | undefined => rules.find((rule) => rule.field === field); + +it("explains created fields by echo, copy, path and slug, leaving constants unlisted", () => { + const notes = ["x", "drafted by the finance team"]; + const result = explainCreate( + created( + { + id: 7, + title: "Quarterly report", + heading: "Quarterly report", + summary: "drafted by the finance team", + projectId: 42, + slug: "quarterly-report", + code: "abc", + status: "open", + }, + { title: "Quarterly report", meta: { notes }, label: "abc" }, + { path: { project: "42" } }, + ), + ); + expect(result.rules).toEqual([ + { field: "id", rule: "identity", generator: "sequence" }, + { field: "title", rule: "echo" }, + { field: "heading", rule: "copy", from: "/title" }, + { field: "summary", rule: "copy", from: "/meta/notes/1" }, + { field: "projectId", rule: "path", from: "project" }, + { field: "slug", rule: "slug", from: "title" }, + ]); + expect(result.tied).toBe(false); +}); + +it("copies only body strings of eight or more characters", () => { + const short = explainCreate( + created({ id: 1, label: "seven77" }, { name: "seven77" }), + ); + expect(ruleFor(short.rules, "label")).toBeUndefined(); + const long = explainCreate( + created({ id: 1, label: "eight888" }, { name: { first: "eight888" } }), + ); + expect(ruleFor(long.rules, "label")).toEqual({ + field: "label", + rule: "copy", + from: "/name/first", + }); +}); + +it.each<[string, JsonValue, number | undefined, boolean]>([ + [ + "ten minutes after", + new Date(START + 10 * MINUTE).toISOString(), + START, + true, + ], + [ + "ten minutes before", + new Date(START - 10 * MINUTE).toISOString(), + START, + true, + ], + [ + "just outside", + new Date(START + 10 * MINUTE + 1).toISOString(), + START, + false, + ], + ["untimed", new Date(START).toISOString(), undefined, false], +])("explains a clock %s the request start", (_, value, startMs, clock) => { + const result = explainCreate(created({ id: 1, at: value }, {}, { startMs })); + expect(ruleFor(result.rules, "at")).toEqual( + clock + ? { + field: "at", + rule: "clock", + format: { kind: "iso", separator: "T", fraction: 3, zone: "Z" }, + } + : undefined, + ); +}); + +it("keeps an epoch clock's unit", () => { + const result = explainCreate(created({ id: 1, at: START / 1000 + 5 }, {})); + expect(ruleFor(result.rules, "at")).toEqual({ + field: "at", + rule: "clock", + format: { kind: "epoch", unit: "s" }, + }); +}); + +it.each<[JsonValue, FieldRule | undefined]>([ + [4, { field: "number", rule: "sequence", scope: "collection" }], + ["4", { field: "number", rule: "sequence", scope: "collection" }], + [10, { field: "number", rule: "sequence", scope: "type" }], + [5, undefined], +])("explains the number %j by the sequence it continues", (value, rule) => { + const collection = [{ number: 1 }, { number: "3" }]; + const evidence = created( + { id: 1, number: value }, + {}, + { + collection, + type: [...collection, { number: 9 }, { number: "x" }], + }, + ); + expect(ruleFor(explainCreate(evidence).rules, "number")).toEqual(rule); +}); + +it.each<[string, JsonObject[], boolean]>([ + ["no other entity shares", [{ ref: OTHER_UUID }, {}], true], + ["another entity shares", [{ ref: OTHER_UUID }, { ref: UUID }], false], + ["the type holds no other entity", [], false], +])("explains a UUID as unique when %s it", (_, type, unique) => { + const result = explainCreate(created({ id: 1, ref: UUID }, {}, { type })); + expect(ruleFor(result.rules, "ref")).toEqual( + unique ? { field: "ref", rule: "unique" } : undefined, + ); +}); + +it.each<[JsonValue, "sequence" | "uuid" | "token"]>([ + [17, "sequence"], + ["17", "sequence"], + [UUID, "uuid"], + ["tok_AbC9", "token"], +])("generates the identity %j as a %s", (id, generator) => { + const result = explainCreate( + created({ key: id, title: "x" }, {}, { idField: "key" }), + ); + expect(result.rules).toEqual([{ field: "key", rule: "identity", generator }]); +}); + +it("rebuilds strings that embed a generated value as a delimited token", () => { + const result = explainCreate( + created( + { + id: "abc123", + number: 4, + url: "https://x.test/items/abc123", + ref: "#4", + near: "abc1234 is not it", + year: "2024", + }, + {}, + { collection: [{ number: 3 }] }, + ), + ); + expect(result.rules).toEqual([ + { field: "id", rule: "identity", generator: "token" }, + { field: "number", rule: "sequence", scope: "collection" }, + { field: "url", rule: "template", from: "id" }, + { field: "ref", rule: "template", from: "number" }, + ]); +}); + +it("accepts echoed members and fixes the others at their sent values", () => { + const result = explainCreate( + created( + { id: 1, title: "A", priority: "high" }, + { title: "A", priority: "HIGH", notify: true }, + ), + ); + expect(result.body).toEqual({ + accepted: ["title"], + fixed: [ + { member: "priority", sent: "HIGH" }, + { member: "notify", sent: true }, + ], + required: ["title", "priority", "notify"], + identity: null, + kinds: [], + }); +}); + +it.each<[string, JsonObject, boolean]>([ + [ + "a constant holds the path value delimited", + { team: "/orgs/acme/teams" }, + true, + ], + ["a constant holds it only inside a word", { team: "acmecorp" }, false], + ["only a path field holds it", { org: "acme" }, false], + [ + "a template keeps it beside the generated value", + { url: "/orgs/acme/items/1" }, + true, + ], +])("ties a create to its collection only when %s", (_, fields, tied) => { + const result = explainCreate( + created({ id: 1, ...fields }, {}, { path: { org: "acme", empty: "" } }), + ); + expect(result.tied).toBe(tied); +}); + +it("explains an update's sent members as echoes, constants or the identity", () => { + const result = explainUpdate( + updated( + { id: "42", title: "Old", status: "open" }, + { id: "42", title: "New", status: "closed" }, + { id: 42, title: "New", status: "CLOSED", comment: "done" }, + ), + ); + expect(result).toEqual({ + rules: [ + { field: "title", rule: "echo" }, + { field: "status", rule: "constant", value: "closed" }, + ], + body: { + accepted: ["title"], + fixed: [ + { member: "status", sent: "CLOSED" }, + { member: "comment", sent: "done" }, + ], + required: [], + identity: "id", + kinds: [], + }, + unexplained: [], + }); +}); + +it("treats omitted read fields as removed only when the selected view covers them", () => { + const before = { + id: "42", + name: "Old", + description: "Kept", + updated_at: "2026-01-01T00:00:00Z", + }; + const after = { id: "42", name: "New" }; + const result = explainUpdate( + updated(before, after, { name: "New" }, { readFields: ["id", "name"] }), + ); + expect(result.unexplained).toEqual([]); + + const complete = explainUpdate( + updated( + before, + after, + { name: "New" }, + { + readFields: ["id", "name", "description", "updated_at"], + }, + ), + ); + expect(complete.unexplained).toEqual(["description", "updated_at"]); +}); + +it("explains unsent changes as a clock, an increment or a slug of an echoed member", () => { + const result = explainUpdate( + updated( + { + id: "42", + title: "Old Title", + slug: "old-title", + version: 3, + revision: "9", + updatedAt: "2026-01-01T00:00:00Z", + color: "red", + }, + { + id: "42", + title: "New Title", + slug: "new-title", + version: 4, + revision: "10", + updatedAt: new Date(START + 1000).toISOString(), + color: "blue", + added: true, + }, + { title: "New Title" }, + ), + ); + expect(result.rules).toEqual([ + { field: "title", rule: "echo" }, + { field: "slug", rule: "slug", from: "title" }, + { field: "version", rule: "increment" }, + { field: "revision", rule: "increment" }, + { + field: "updatedAt", + rule: "clock", + format: { kind: "iso", separator: "T", fraction: 3, zone: "Z" }, + }, + ]); + expect(result.unexplained).toEqual(["color"]); +}); + +it("leaves a slug of a transformed member, an untimed clock and a jump unexplained", () => { + const result = explainUpdate( + updated( + { id: "42", title: "Old", slug: "old", at: "2026-01-01T00:00:00Z", n: 3 }, + { + id: "42", + title: "new title", + slug: "new-title", + at: new Date(START).toISOString(), + n: 5, + }, + { title: "New Title" }, + { startMs: undefined }, + ), + ); + expect(result.rules).toEqual([ + { field: "title", rule: "constant", value: "new title" }, + ]); + expect(result.unexplained).toEqual(["slug", "at", "n"]); +}); + +it.each<[string, string[]]>([ + ["PUT", ["title", "status"]], + ["PATCH", []], + ["POST", []], +])( + "requires the members a %s sent apart from the identity", + (method, required) => { + const after = { id: "42", title: "New", status: "closed" }; + const body = { id: "42", title: "New", status: "closed" }; + const result = explainUpdate(updated(after, after, body, { method })); + expect(result.body.required).toEqual(required); + expect(result.body.identity).toBe("id"); + }, +); + +type Shaped = [WriteResponse["shape"], string | null]; + +it.each<[string, JsonValue | undefined, boolean, Shaped]>([ + ["no body", undefined, true, ["empty", null]], + ["an empty response with a body", { id: "42" }, true, ["empty", null]], + ["no recorded body", undefined, false, ["recorded", null]], + ["the entity at the root", { id: "42", title: "x" }, false, ["entity", ""]], + [ + "the entity in one member", + { data: { id: 42 } }, + false, + ["entity", "/data"], + ], + [ + "the entity in a one-element array", + [{ id: "42" }], + false, + ["entity", "/0"], + ], + ["the identity at the root", "42", false, ["identity", ""]], + ["a numeric identity at the root", 42, false, ["identity", ""]], + [ + "the identity in one member", + { created: "42", ok: true }, + false, + ["identity", "/created"], + ], + [ + "the identity in an escaped member", + { "a/b": 42 }, + false, + ["identity", "/a~1b"], + ], + [ + "the identity in two members", + { a: "42", b: "42" }, + false, + ["recorded", null], + ], + ["no identity", { ok: true }, false, ["recorded", null]], +])("shapes a response carrying %s", (_, body, empty, [shape, pointer]) => { + expect(responseShapeOf(3, body, empty, { idField: "id", id: "42" })).toEqual({ + ordinal: 3, + shape, + pointer, + }); +}); + +it.each<[JsonValue | undefined, JsonValue | undefined, number | undefined]>([ + [2, 10, -1], + ["10", 9, 1], + ["7", 7, 0], + ["2026-03-04T05:00:00Z", "2026-03-04T06:30:00+02:00", 1], + [Date.UTC(2026, 2, 4, 6) / 1000, "2026-03-04T05:00:00Z", 1], + ["apple", "banana", -1], + ["Zebra", "apple", -1], + ["b", "b", 0], + [5, "apple", undefined], + [true, false, undefined], + [undefined, "x", undefined], + [{ a: 1 }, { a: 1 }, undefined], +])("orders %j against %j as %j", (left, right, sign) => { + const order = compareValues(left, right); + expect(order === undefined ? undefined : Math.sign(order)).toBe(sign); +}); + +it("offers each strictly ordered field in its direction, then last and first", () => { + const members = [ + { id: 1, at: "2026-03-02T00:00:00Z", name: "b", tag: "x" }, + { id: 2, at: "2026-03-01T12:00:00+02:00", name: "a", tag: "x" }, + { id: "3", at: "2026-02-28T00:00:00Z", name: "c" }, + ]; + expect(insertCandidates(members, ["id", "at", "name", "tag"])).toEqual([ + { rule: "sorted", field: "id", direction: "ascending" }, + { rule: "sorted", field: "at", direction: "descending" }, + { rule: "last" }, + { rule: "first" }, + ]); + const fallback = [{ rule: "last" }, { rule: "first" }]; + expect(insertCandidates([{ id: 1 }], ["id"])).toEqual(fallback); + expect(insertCandidates([], ["id"])).toEqual(fallback); +}); + +const ASCENDING = [{ n: 1 }, { n: 3 }, { n: 5 }]; +const DESCENDING = [{ n: 5 }, { n: 3 }, { n: 1 }]; +const up: InsertRule = { rule: "sorted", field: "n", direction: "ascending" }; +const down: InsertRule = { + rule: "sorted", + field: "n", + direction: "descending", +}; + +it.each<[string, InsertRule, JsonObject, number]>([ + ["first", { rule: "first" }, { n: 9 }, 0], + ["last", { rule: "last" }, { n: 0 }, 3], + ["ascending, between members", up, { n: 4 }, 2], + ["ascending, before every member", up, { n: 0 }, 0], + ["ascending, after every member", up, { n: 6 }, 3], + ["ascending, after an equal member", up, { n: 3 }, 2], + ["descending, between members", down, { n: 4 }, 1], + ["descending, after every member", down, { n: 0 }, 3], + ["sorted, with an incomparable value", up, { n: true }, 3], + ["sorted, without the field", up, {}, 3], +])("inserts %s", (_, rule, fields, index) => { + const members = rule === down ? DESCENDING : ASCENDING; + expect(insertIndex(rule, members, fields)).toBe(index); +}); + +it.each<[string, JsonValue, JsonObject, string | undefined]>([ + ["the root", { theme: "dark", lang: "en" }, { theme: "light" }, ""], + [ + "the member sharing most", + { theme: "dark", settings: { theme: "dark", lang: "en" } }, + { theme: "light", lang: "fr" }, + "/settings", + ], + [ + "the root on a tie", + { theme: "d", s: { lang: "en" } }, + { theme: "x", lang: "y" }, + "", + ], + ["an escaped member", { "a/b": { theme: "dark" } }, { theme: "x" }, "/a~1b"], + [ + "nothing when none is shared", + { a: 1, b: { c: 2 } }, + { theme: "x" }, + undefined, + ], + ["nothing in an array", [{ theme: "dark" }], { theme: "x" }, undefined], +])("points a document write at %s", (_, value, body, pointer) => { + expect(documentPointerOf(value, body)).toBe(pointer); +}); + +it.each<[string, string, WriteEffect, string | null]>([ + ["POST", "/users", "create", "/users"], + ["POST", "/users/", "create", "/users"], + ["POST", "/users/{userId}/posts", "create", "/users/{userId}/posts"], + ["PUT", "/users/{userId}", "update", "/users"], + ["PATCH", "/users/{id}", "update", "/users"], + ["POST", "/users/{id}", "update", "/users"], + ["DELETE", "/users/{id}", "delete", "/users"], + [ + "DELETE", + "/users/{userId}/posts/{postId}", + "delete", + "/users/{userId}/posts", + ], + ["GET", "/users/{id}", "none", null], + ["DELETE", "/users", "none", null], + ["PUT", "/users", "none", null], + ["PUT", "/users/me", "none", null], + ["POST", "/users/{id}/archive", "none", null], + ["PUT", "/settings", "document", "get_settings"], + ["PATCH", "/settings/", "document", "get_settings"], + ["POST", "/settings", "document", "get_settings"], + ["DELETE", "/settings", "none", null], +])("classifies %s %s as %s", (method, path, effect, target) => { + expect( + classifyWrite( + { method, path_template: path }, + new Set(["/users", "/users/{userId}/posts"]), + new Map([["/settings", "get_settings"]]), + ), + ).toEqual({ effect, target }); +}); + +it.each<[string, string, Record | undefined]>([ + [ + "/orgs/{org}/repos/{repo}", + "/orgs/acme/repos/rea", + { org: "acme", repo: "rea" }, + ], + ["/orgs/{org}", "/orgs/a%20b%2Fc/", { org: "a b/c" }], + ["/orgs/{org}", "/orgs/%E0%A4%A", { org: "%E0%A4%A" }], + ["/orgs", "/orgs", {}], + ["/orgs/{org}", "/teams/acme", undefined], + ["/orgs/{org}", "/orgs/acme/repos", undefined], +])("reads %s from %s", (template, path, values) => { + expect(pathValuesOf(template, path)).toEqual( + values === undefined ? undefined : new Map(Object.entries(values)), + ); +}); + +it.each<[string, string, boolean]>([ + ["/users", "/users/{id}", true], + ["/users/{id}/posts", "/users/42", true], + ["/users/42/posts", "/users/{id}/posts/7", true], + ["/", "/anything/at/all", true], + ["/users/42", "/users/43", false], + ["/users", "/teams/{id}", false], + ["/{tenant}/users", "/acme/teams", false], +])("relates %s and %s by prefix: %s", (left, right, related) => { + expect(prefixRelated(left, right)).toBe(related); + expect(prefixRelated(right, left)).toBe(related); +}); + +it( + "explains many copied and slugged fields of a wide body without searching it again for each", + { timeout: 5_000 }, + () => { + const n = 8_000; + const value = (index: number) => `value-${String(index).padStart(6, "0")}`; + const nested = Object.fromEntries( + Array.from({ length: n }, (_, index) => [ + `g${String(index)}`, + { s: value(index) }, + ]), + ); + const copies = Object.fromEntries( + Array.from({ length: n }, (_, index) => [ + `c${String(index)}`, + value(index), + ]), + ); + const copied = explainCreate(created({ id: 1, ...copies }, nested)); + expect(ruleFor(copied.rules, `c${String(n - 1)}`)).toEqual({ + field: `c${String(n - 1)}`, + rule: "copy", + from: `/g${String(n - 1)}/s`, + }); + // Strings no member slugs to, in a body of many members, each tried as a slug. + const flat = Object.fromEntries( + Array.from({ length: n }, (_, index) => [ + `m${String(index)}`, + `Member ${String(index)}`, + ]), + ); + const others = Object.fromEntries( + Array.from({ length: n }, (_, index) => [ + `o${String(index)}`, + `x${String(index)}`, + ]), + ); + const slugged = explainCreate( + created({ id: 1, ...others, s: "member-7" }, flat), + ); + expect(ruleFor(slugged.rules, "s")).toEqual({ + field: "s", + rule: "slug", + from: "m7", + }); + }, +); + +it( + "searches a wide request body for a copied value in linear time", + { timeout: 5_000 }, + () => { + // Shifting each value off the queue made one fruitless search over 500,000 zeros take about a minute. + const wide = Array.from({ length: 500_000 }, () => 0); + const result = explainCreate( + created({ id: 1, note: "not anywhere in the body" }, { items: wide }), + ); + expect(ruleFor(result.rules, "note")).toBeUndefined(); + }, +); diff --git a/src/domain/analysisView/environmentStateWriteExplain.ts b/src/domain/analysisView/environmentStateWriteExplain.ts new file mode 100644 index 0000000000..6cafd4522d --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteExplain.ts @@ -0,0 +1,544 @@ +import type { JsonValue } from "../jsonValue.js"; +import { compareUnicodeCodePoints as compare } from "../unicodeCodePointOrder.js"; +import { storedKinds } from "./environmentStateWriteApply.js"; +import { + isObject, + locateItem, + pointerTo, + same, + scalarText, + type JsonObject, +} from "./environmentStateRules.js"; +import type { + FieldRule, + InsertRule, + StateWritePlan, + WriteBody, + WriteCall, + WriteResponse, +} from "./environmentStateWriteRules.js"; +import { + containsDelimited, + delimitedOffsets, + integerOf, + isUuid, + readClock, + slugOf, +} from "./environmentStateWriteValues.js"; + +/** A clock value counts as written by the request when it lies this close to the request's start. */ +const CLOCK_WINDOW_MS = 10 * 60_000; +/** Shorter body strings are too common to show that a field copied one. */ +const COPY_MIN_LENGTH = 8; + +/** What explaining a create reads: the observed entity, the call that wrote it and the entities it joined. */ +export interface CreateEvidence { + readonly entity: JsonObject; + readonly call: WriteCall & { readonly body: JsonObject }; + readonly idField: string; + /** The request's start in epoch milliseconds, when the timeline is timed. */ + readonly startMs: number | undefined; + readonly collection: readonly JsonObject[]; + readonly type: readonly JsonObject[]; +} + +/** What explaining a create looks values up in, built once per request body. */ +interface BodyIndex { + /** Each body string long enough to show a copy, at its shallowest pointer (breadth-first, members in order). */ + readonly copies: ReadonlyMap; + /** Each slug a top-level string member slugs to, other than its own value, from the first such member. */ + readonly slugs: ReadonlyMap; +} + +/** Index a request body once, so explaining each field looks values up rather than walking the body again. */ +const indexBody = (body: JsonObject): BodyIndex => { + const copies = new Map(); + const pending: { readonly value: JsonValue; readonly pointer: string }[] = [ + { value: body, pointer: "" }, + ]; + // An index walks the queue: shifting a wide body's values one by one would take quadratic time. + for (let at = 0; at < pending.length; at += 1) { + const next = pending[at]!; + if (typeof next.value === "string") { + if (next.value.length >= COPY_MIN_LENGTH && !copies.has(next.value)) + copies.set(next.value, next.pointer); + continue; + } + const children: [string, JsonValue][] = Array.isArray(next.value) + ? next.value.map((item, index) => [String(index), item]) + : isObject(next.value) + ? Object.entries(next.value) + : []; + for (const [key, item] of children) + pending.push({ + value: item, + pointer: `${next.pointer}${pointerTo(key)}`, + }); + } + const slugs = new Map(); + for (const [member, sent] of Object.entries(body)) + if (typeof sent === "string") { + const slug = slugOf(sent); + if (slug !== sent && !slugs.has(slug)) slugs.set(slug, member); + } + return { copies, slugs }; +}; + +const isClockNear = ( + value: JsonValue, + startMs: number | undefined, +): boolean => { + const clock = readClock(value); + return ( + clock !== undefined && + startMs !== undefined && + Math.abs(clock.ms - startMs) <= CLOCK_WINDOW_MS + ); +}; + +/** One more than the largest integer the members hold in the field, or undefined when none holds one. */ +export const nextInteger = ( + members: readonly JsonObject[], + field: string, +): number | undefined => { + let largest: number | undefined; + for (const member of members) { + const value = integerOf(member[field]); + if (value !== undefined && (largest === undefined || value > largest)) + largest = value; + } + return largest === undefined ? undefined : largest + 1; +}; + +const identityRule = (field: string, value: JsonValue): FieldRule => ({ + field, + rule: "identity", + generator: + integerOf(value) !== undefined + ? "sequence" + : isUuid(value) + ? "uuid" + : "token", +}); + +/** The rule a request explains a created field by, before generated values and templates are considered. */ +const requestRule = ( + evidence: CreateEvidence, + index: BodyIndex, + field: string, + value: JsonValue, +): FieldRule | undefined => { + const { body, pathValues } = evidence.call; + if (Object.hasOwn(body, field) && same(body[field], value)) + return { field, rule: "echo" }; + if (typeof value === "string" && value.length >= COPY_MIN_LENGTH) { + const from = index.copies.get(value); + if (from !== undefined) return { field, rule: "copy", from }; + } + const text = scalarText(value); + for (const [name, pathValue] of pathValues) + if (text !== undefined && text === pathValue) + return { field, rule: "path", from: name }; + const slugged = typeof value === "string" && value.length >= 3; + const member = slugged ? index.slugs.get(value) : undefined; + return member === undefined + ? undefined + : { field, rule: "slug", from: member }; +}; + +/** The rule a created field's generated value follows, if it follows one. */ +const generatedRule = ( + evidence: CreateEvidence, + field: string, + value: JsonValue, +): FieldRule | undefined => { + if (field === evidence.idField) return identityRule(field, value); + const clock = readClock(value); + if (clock !== undefined && isClockNear(value, evidence.startMs)) + return { field, rule: "clock", format: clock.format }; + const integer = integerOf(value); + if (integer !== undefined) { + if (nextInteger(evidence.collection, field) === integer) + return { field, rule: "sequence", scope: "collection" }; + if (nextInteger(evidence.type, field) === integer) + return { field, rule: "sequence", scope: "type" }; + } + if ( + isUuid(value) && + evidence.type.length > 0 && + evidence.type.every((member) => member[field] !== value) + ) + return { field, rule: "unique" }; + return undefined; +}; + +/** A string that embeds a generated value delimited is rebuilt around the new value. */ +const templateRule = ( + entity: JsonObject, + field: string, + generated: readonly FieldRule[], +): FieldRule | undefined => { + const value = entity[field]; + if (typeof value !== "string") return undefined; + for (const rule of generated) { + const token = scalarText(entity[rule.field]); + if ( + rule.field !== field && + token !== undefined && + token.length > 0 && + delimitedOffsets(value, token).length > 0 + ) + return { field, rule: "template", from: rule.field }; + } + return undefined; +}; + +/** Body members the application stored as sent, and the others with what they were sent. */ +const bodyOf = ( + body: JsonObject, + echoed: ReadonlySet, + required: readonly string[], +): WriteBody => ({ + accepted: Object.keys(body).filter((member) => echoed.has(member)), + fixed: Object.entries(body) + .filter(([member]) => !echoed.has(member)) + .map(([member, sent]) => ({ member, sent })), + required: [...required], + identity: null, + kinds: [], +}); + +/** Explain every field of a created entity: echoed, copied, path, slug, generated, template or constant (unlisted). */ +export const explainCreate = ( + evidence: CreateEvidence, +): { + readonly rules: FieldRule[]; + readonly body: WriteBody; + readonly tied: boolean; +} => { + const rules: FieldRule[] = []; + const unexplained: string[] = []; + const index = indexBody(evidence.call.body); + for (const [field, value] of Object.entries(evidence.entity)) { + const rule = + requestRule(evidence, index, field, value) ?? + generatedRule(evidence, field, value); + if (rule === undefined) unexplained.push(field); + else rules.push(rule); + } + const generated = rules.filter( + (rule) => + rule.rule === "identity" || + rule.rule === "sequence" || + rule.rule === "unique", + ); + const constants: string[] = []; + for (const field of unexplained) { + const template = templateRule(evidence.entity, field, generated); + if (template === undefined) constants.push(field); + else rules.push(template); + } + const echoed = new Set( + rules.filter((rule) => rule.rule === "echo").map((rule) => rule.field), + ); + const tokens = [...evidence.call.pathValues.values()].filter( + (value) => value.length > 0, + ); + return { + rules, + body: bodyOf(evidence.call.body, echoed, Object.keys(evidence.call.body)), + // A template rebuilds only its generated value, so other text it keeps ties it like a constant. + tied: [ + ...constants, + ...rules.flatMap((rule) => + rule.rule === "template" ? [rule.field] : [], + ), + ].some((field) => + tokens.some((token) => containsDelimited(evidence.entity[field]!, token)), + ), + }; +}; + +/** What explaining an update reads: the entity before and after, the call and its method. */ +export interface UpdateEvidence { + readonly before: JsonObject; + readonly after: JsonObject; + readonly call: WriteCall & { readonly body: JsonObject }; + readonly method: string; + readonly idField: string; + readonly id: string; + readonly startMs: number | undefined; + /** Fields covered by a later read view. Missing fields outside that coverage remain unknown. */ + readonly readFields?: readonly string[]; + /** The fields of the observing read's view when that view showed the entity before, so one the entity lacked was added. */ + readonly shown?: readonly string[]; +} + +/** The rule a field the update changed without sending follows, if any; `slugs` maps each echoed member's slug to the first such member. */ +const changeRule = ( + evidence: UpdateEvidence, + field: string, + slugs: ReadonlyMap, +): FieldRule | undefined => { + const before = evidence.before[field]; + const after = evidence.after[field]!; + if (readClock(after) !== undefined && isClockNear(after, evidence.startMs)) + return { field, rule: "clock", format: readClock(after)!.format }; + const was = integerOf(before); + if (was !== undefined && integerOf(after) === was + 1) + return { field, rule: "increment" }; + const member = typeof after === "string" ? slugs.get(after) : undefined; + return member === undefined + ? undefined + : { field, rule: "slug", from: member }; +}; + +/** Each slug an echoed string member slugs to, from the first such member. */ +const echoedSlugs = ( + body: JsonObject, + echoed: ReadonlySet, +): Map => { + const slugs = new Map(); + for (const member of echoed) { + const sent = body[member]; + if (typeof sent === "string" && !slugs.has(slugOf(sent))) + slugs.set(slugOf(sent), member); + } + return slugs; +}; + +/** Explain an update: echoed members, fixed members whose stored value is constant, and the changes it made unsent. */ +export const explainUpdate = ( + evidence: UpdateEvidence, +): { + readonly rules: FieldRule[]; + readonly body: WriteBody; + readonly unexplained: string[]; +} => { + const { body } = evidence.call; + const rules: FieldRule[] = []; + const echoed = new Set(); + const unexplained: string[] = []; + let identity: string | null = null; + for (const [member, sent] of Object.entries(body)) { + // The identity is the entity's key: the path's id repeated is the identity member; another value is one the application ignored, or used to re-key the entity, which no rule models. + if (member === evidence.idField) { + if (scalarText(sent) === evidence.id) identity = member; + else if (!same(evidence.after[member], evidence.before[member])) + unexplained.push(member); + continue; + } + if (!Object.hasOwn(evidence.after, member)) continue; + if (same(evidence.after[member], sent)) { + echoed.add(member); + rules.push({ field: member, rule: "echo" }); + } else + rules.push({ + field: member, + rule: "constant", + value: evidence.after[member]!, + }); + } + // Serving keeps every field an update does not write, so one a read shows the update removed is a change no rule models. + if (evidence.readFields !== undefined) + for (const field of Object.keys(evidence.before)) + if ( + evidence.readFields.includes(field) && + !Object.hasOwn(evidence.after, field) + ) + unexplained.push(field); + const slugs = echoedSlugs(body, echoed); + for (const field of Object.keys(evidence.after)) { + if ( + Object.hasOwn(body, field) || + (!Object.hasOwn(evidence.before, field) && + !(evidence.shown ?? []).includes(field)) || + same(evidence.before[field], evidence.after[field]) + ) + continue; + const rule = changeRule(evidence, field, slugs); + if (rule === undefined) unexplained.push(field); + else rules.push(rule); + } + const shaped = bodyOf( + Object.fromEntries( + Object.entries(body).filter(([member]) => member !== identity), + ), + echoed, + evidence.method === "PUT" + ? Object.keys(body).filter((member) => member !== identity) + : [], + ); + return { rules, body: { ...shaped, identity }, unexplained }; +}; + +/** + * How a write's response carries the entity it wrote: an object with the type's identity field + * equal to the id (entity), the id alone at the root or in one member (identity), nothing + * (empty), or anything else (recorded). + */ +export const responseShapeOf = ( + ordinal: number, + body: JsonValue | undefined, + empty: boolean, + entity: { readonly idField: string; readonly id: string }, +): WriteResponse => { + if (empty) return { ordinal, shape: "empty", pointer: null }; + if (body === undefined) return { ordinal, shape: "recorded", pointer: null }; + const located = locateItem(body, entity.id); + if (located?.keys.includes(entity.idField) === true) + return { ordinal, shape: "entity", pointer: located.pointer }; + if (scalarText(body) === entity.id) + return { ordinal, shape: "identity", pointer: "" }; + if (isObject(body)) { + const members = Object.entries(body).filter( + ([, value]) => scalarText(value) === entity.id, + ); + if (members.length === 1) + return { + ordinal, + shape: "identity", + pointer: pointerTo(members[0]![0]), + }; + } + return { ordinal, shape: "recorded", pointer: null }; +}; + +/** Order two field values as numbers, instants or text; undefined when they are not comparable. */ +export const compareValues = ( + left: JsonValue | undefined, + right: JsonValue | undefined, +): number | undefined => { + const [a, b] = [integerOf(left), integerOf(right)]; + if (a !== undefined && b !== undefined) return a - b; + const [c, d] = [readClock(left), readClock(right)]; + if (c !== undefined && d !== undefined) return c.ms - d.ms; + return typeof left === "string" && typeof right === "string" + ? compare(left, right) + : undefined; +}; + +/** Whether the members are strictly ordered by the field in the direction. */ +const orderedBy = ( + members: readonly JsonObject[], + field: string, + direction: "ascending" | "descending", +): boolean => + members.every((member, index) => { + if (index === 0) return true; + const order = compareValues(members[index - 1]![field], member[field]); + return ( + order !== undefined && (direction === "ascending" ? order < 0 : order > 0) + ); + }); + +/** Insert rules consistent with a collection's recorded order: sorted by identity or clock fields when shown, then last and first. */ +export const insertCandidates = ( + members: readonly JsonObject[], + fields: readonly string[], +): InsertRule[] => { + const sorted: InsertRule[] = []; + if (members.length >= 2) + for (const field of fields) + for (const direction of ["ascending", "descending"] as const) + if (orderedBy(members, field, direction)) + sorted.push({ rule: "sorted", field, direction }); + return [...sorted, { rule: "last" }, { rule: "first" }]; +}; + +/** Where an entity with these fields enters a collection under an insert rule. */ +export const insertIndex = ( + rule: InsertRule, + members: readonly JsonObject[], + fields: JsonObject, +): number => { + if (rule.rule !== "sorted") return rule.rule === "first" ? 0 : members.length; + const index = members.findIndex((member) => { + const order = compareValues(fields[rule.field], member[rule.field]); + return ( + order !== undefined && + (rule.direction === "ascending" ? order < 0 : order > 0) + ); + }); + return index < 0 ? members.length : index; +}; + +/** The object in a document value, at the root or in one member, that holds most of a body's members. */ +export const documentPointerOf = ( + value: JsonValue, + body: JsonObject, +): string | undefined => { + const members = Object.keys(body); + const candidates: [string, JsonValue][] = [ + ["", value], + ...(isObject(value) + ? Object.entries(value).map(([name, item]): [string, JsonValue] => [ + pointerTo(name), + item, + ]) + : []), + ]; + let best: { readonly pointer: string; readonly shared: number } | undefined; + for (const [pointer, candidate] of candidates) { + if (!isObject(candidate)) continue; + const shared = members.filter((member) => + Object.hasOwn(candidate, member), + ).length; + if (shared > 0 && (best === undefined || shared > best.shared)) + best = { pointer, shared }; + } + return best?.pointer; +}; + +/** A plan's body with the members and fixed values a later call's body adds, and the JSON types it stored those members as when the observation shows none. */ +export const refinedBody = ( + body: WriteBody, + later: WriteBody, + values: { readonly observed: JsonObject; readonly stored: JsonObject }, +): WriteBody => { + const fixed = new Set(body.fixed.map((entry) => entry.member)); + const accepted = later.accepted.filter( + (member) => !body.accepted.includes(member), + ); + return { + ...body, + // A later call may be the first to show the body repeating the path's id. + identity: body.identity ?? later.identity, + accepted: [...body.accepted, ...accepted], + fixed: [ + ...body.fixed, + ...later.fixed.filter((entry) => !fixed.has(entry.member)), + ], + // Serving reads member types from the observation, which may not show a member a later call stored. + kinds: [ + ...body.kinds, + ...storedKinds(accepted, values.observed, values.stored), + ], + }; +}; + +/** Whether a later call explains a member or field otherwise than the plan: echoed by one and stored as something else by the other, or ruled differently. */ +export const conflictsWithPlan = ( + plan: Pick, + explained: ReturnType, +): boolean => { + const body = plan.body!; + const fixed = new Map(body.fixed.map((entry) => [entry.member, entry.sent])); + const rules = new Map(plan.rules.map((rule) => [rule.field, rule])); + const { identity } = explained.body; + // Serving checks a call's identity member before its fixed values, so a member both cannot take every recorded call. + return ( + explained.body.accepted.some((member) => fixed.has(member)) || + explained.body.fixed.some( + (entry) => + body.accepted.includes(entry.member) || + entry.member === body.identity || + (fixed.has(entry.member) && !same(fixed.get(entry.member), entry.sent)), + ) || + (identity !== null && fixed.has(identity)) || + explained.rules.some((rule) => { + const held = rules.get(rule.field); + return held !== undefined && !same(held, rule); + }) + ); +}; diff --git a/src/domain/analysisView/environmentStateWriteObserve.ts b/src/domain/analysisView/environmentStateWriteObserve.ts new file mode 100644 index 0000000000..ac79d7e01d --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteObserve.ts @@ -0,0 +1,198 @@ +import type { JsonValue } from "../jsonValue.js"; +import { overlapsWrite } from "./environmentStateWriteAttribution.js"; +import { + collectionPathOf, + isObject, + lastSegmentValue, + listItems, + pointerTo, + scalarText, + type JsonObject, + type StateEntity, + type StateRecordFacts, + type StateView, +} from "./environmentStateRules.js"; + +/** Where an observed entity object was found: the record and a JSON pointer into its response. */ +export interface Observed { + readonly ordinal: number; + readonly pointer: string; + readonly object: JsonObject; + readonly id: string; +} + +/** Objects a response carries where an entity may be, in precedence tiers: the root, one member, then the element of a one-element array at the root or in one member. */ +const entitySlots = (body: JsonValue): [string, JsonValue][][] => { + const members = isObject(body) ? Object.entries(body) : []; + const elements: [string, JsonValue][] = + Array.isArray(body) && body.length === 1 ? [["/0", body[0]!]] : []; + for (const [name, item] of members) + if (Array.isArray(item) && item.length === 1) + elements.push([`${pointerTo(name)}/0`, item[0]!]); + return [ + [["", body]], + members.map(([name, item]): [string, JsonValue] => [pointerTo(name), item]), + elements, + ]; +}; + +/** The entity object a response carries with an identity value the state does not hold yet: the only one in the first tier that has one. */ +export const newEntityIn = ( + ordinal: number, + body: JsonValue | undefined, + idField: string, + held: (id: string) => boolean, +): Observed | undefined => { + if (body === undefined) return undefined; + for (const tier of entitySlots(body)) { + const found = tier.flatMap(([pointer, object]) => { + const id = isObject(object) ? scalarText(object[idField]) : undefined; + // An object holding only the identity names the entity without showing it. + return isObject(object) && + id !== undefined && + !held(id) && + Object.keys(object).some((key) => key !== idField) + ? [{ ordinal, pointer, object, id }] + : []; + }); + if (found.length > 0) return found.length === 1 ? found[0] : undefined; + } + return undefined; +}; + +/** The fields an item or list view shows; undefined for any other view. */ +export const viewFields = ( + view: StateView | undefined, +): readonly string[] | undefined => + view?.shape === "item" || view?.shape === "list" ? view.fields : undefined; + +/** The fields of the view a read observed an entity with, when that view had shown the entity before. */ +export const shownBefore = ( + viewOf: (ordinal: number) => StateView | undefined, + entity: Pick, + ordinal: number, +): readonly string[] => { + const view = viewOf(ordinal); + const shown = entity.record_ordinals.some( + (earlier) => viewOf(earlier)?.tool === view?.tool, + ); + return (shown ? viewFields(view) : undefined) ?? []; +}; + +/** What a forward search for an observation reads: records after a position, read views and response bodies. */ +export interface ObservationSearch { + readonly records: readonly StateRecordFacts[]; + readonly from: number; + /** A timed write's completion. A read begun while it still ran may show the prior state. */ + readonly after?: number; + readonly viewOf: (ordinal: number) => StateView | undefined; + readonly withBody: ( + ordinal: number, + use: (body: JsonValue | undefined) => T, + ) => T; +} + +const succeeded = (record: StateRecordFacts): boolean => + record.status !== null && record.status >= 200 && record.status < 300; + +/** + * The first later read, before the next write, that shows a written entity: an item read of + * `id` when given, else an item read of an unheld id or a collection read listing exactly one. + * A read begun while the write still ran may show the earlier state, so it is not chosen. + */ +export const laterObservation = ( + search: ObservationSearch, + target: { + readonly type: string; + readonly collection: string; + readonly idField: string; + readonly id?: string; + readonly held: (id: string) => boolean; + readonly isWrite: (record: StateRecordFacts) => boolean; + /** An item read matches by this instead of equal collection strings. */ + readonly sameItem?: (record: StateRecordFacts) => boolean; + /** Skip read views that do not cover the facts needed by this observation. */ + readonly covers?: ( + view: Extract, + ) => boolean; + }, +): Observed | undefined => { + for (const record of search.records.slice(search.from + 1)) { + if (target.isWrite(record)) return undefined; + // The same overlap the check path ignores: the write was still running when this read began. + if (search.after !== undefined && overlapsWrite(search.after, record.start)) + continue; + const view = search.viewOf(record.ordinal); + if ( + !succeeded(record) || + (view?.shape !== "item" && view?.shape !== "list") || + view.type !== target.type + ) + continue; + if (target.covers !== undefined && !target.covers(view)) continue; + const found = + view.shape === "item" + ? itemObservation(search, record, view.pointer, target) + : listObservation(search, record, view.pointer, target); + if (found !== undefined) return found; + } + return undefined; +}; + +const itemObservation = ( + search: ObservationSearch, + record: StateRecordFacts, + pointer: string, + target: Parameters[1], +): Observed | undefined => { + const pathId = lastSegmentValue(record.path); + const named = + target.sameItem?.(record) ?? + (collectionPathOf(record, true) === target.collection && + (target.id === undefined ? !target.held(pathId) : pathId === target.id)); + if (!named) return undefined; + const id = target.id ?? pathId; + return search.withBody(record.ordinal, (body) => { + const object = + body === undefined + ? undefined + : entitySlots(body) + .flat() + .find(([at]) => at === pointer)?.[1]; + return isObject(object) && scalarText(object[target.idField]) === id + ? { ordinal: record.ordinal, pointer, object, id } + : undefined; + }); +}; + +const listObservation = ( + search: ObservationSearch, + record: StateRecordFacts, + pointer: string, + target: Parameters[1], +): Observed | undefined => { + if (collectionPathOf(record, false) !== target.collection) return undefined; + return search.withBody(record.ordinal, (body) => { + const items = body === undefined ? undefined : listItems(body, pointer); + const unheld = (items ?? []) + .map((object, index) => ({ + object, + index, + id: scalarText(object[target.idField]), + })) + .filter( + (entry): entry is { object: JsonObject; index: number; id: string } => + entry.id !== undefined && + !target.held(entry.id) && + (target.id === undefined || entry.id === target.id), + ); + return unheld.length === 1 + ? { + ordinal: record.ordinal, + pointer: `${pointer}/${String(unheld[0]!.index)}`, + object: unheld[0]!.object, + id: unheld[0]!.id, + } + : undefined; + }); +}; diff --git a/src/domain/analysisView/environmentStateWriteRules.ts b/src/domain/analysisView/environmentStateWriteRules.ts new file mode 100644 index 0000000000..bf315fee74 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteRules.ts @@ -0,0 +1,210 @@ +import type { JsonValue } from "../jsonValue.js"; +import { + isParam, + normalizedPath, + segmentsOf, + type JsonObject, +} from "./environmentStateRules.js"; +import type { ClockFormat } from "./environmentStateWriteValues.js"; + +/** What a write tool does to the state, by method and the target its path names. */ +export type WriteEffect = "create" | "update" | "delete" | "document" | "none"; + +/** Why a write tool is not served. */ +export type WriteReason = + | "no-state-target" + | "no-call" + | "body-unusable" + | "no-observation" + | "unexplained-change" + | "contradicted" + | "side-effect" + | "unconfirmed" + | "incomplete-entities"; + +/** Rules that generate a value when served; the gate takes the recorded value instead. */ +export type GeneratedRule = + | { + readonly field: string; + readonly rule: "identity"; + readonly generator: "sequence" | "uuid" | "token"; + } + | { + readonly field: string; + readonly rule: "sequence"; + readonly scope: "collection" | "type"; + } + | { readonly field: string; readonly rule: "unique" } + | { + readonly field: string; + readonly rule: "clock"; + readonly format: ClockFormat; + }; + +/** How one field of a written entity gets its value; fields no rule names keep the observed value. */ +export type FieldRule = + | GeneratedRule + | { readonly field: string; readonly rule: "echo" } + | { readonly field: string; readonly rule: "copy"; readonly from: string } + | { readonly field: string; readonly rule: "path"; readonly from: string } + | { readonly field: string; readonly rule: "slug"; readonly from: string } + | { readonly field: string; readonly rule: "template"; readonly from: string } + | { readonly field: string; readonly rule: "increment" } + /** An update stores `value` whenever the call sends the member, as the call that showed it did. */ + | { + readonly field: string; + readonly rule: "constant"; + readonly value: JsonValue; + }; + +/** Where a create places its entity in its collection's order. */ +export type InsertRule = + | { readonly rule: "first" | "last" } + | { + readonly rule: "sorted"; + readonly field: string; + readonly direction: "ascending" | "descending"; + }; + +/** The JSON type of a stored value that is not null. */ +export type JsonKind = "array" | "boolean" | "number" | "object" | "string"; + +/** The body members a write accepts: echoed ones with any value, fixed ones only with the value they were sent. */ +export interface WriteBody { + readonly accepted: string[]; + readonly fixed: { readonly member: string; readonly sent: JsonValue }[]; + readonly required: string[]; + /** The member that may repeat the target's identity, which must then equal the path's. */ + readonly identity: string | null; + /** The JSON type a later call stored an accepted member as, for members the observation does not show. */ + readonly kinds: { readonly member: string; readonly kind: JsonKind }[]; +} + +/** What a write answers: the written entity or its identity rendered into a recorded body, no body, or the recorded body. */ +export interface WriteResponse { + readonly ordinal: number; + readonly shape: "entity" | "identity" | "empty" | "recorded"; + readonly pointer: string | null; +} + +/** One write tool's modelled effect and whether the capture confirmed it. */ +export interface StateWritePlan { + readonly tool: string; + readonly effect: WriteEffect; + /** The entity type for entity effects; the document tool for a document write. */ + readonly target: string | null; + readonly served: boolean; + readonly reason: WriteReason | null; + readonly exemplar_ordinal: number | null; + /** The record whose response showed the written entity or document, and where. */ + readonly observation: { + readonly ordinal: number; + readonly pointer: string; + } | null; + readonly response: WriteResponse | null; + readonly body: WriteBody | null; + readonly rules: FieldRule[]; + /** A create tied to the exemplar's collection writes only there. */ + readonly collection: string | null; + readonly insert: InsertRule | null; + readonly merge: "merge" | "replace" | null; + readonly checked_record_ordinals: number[]; + readonly contradicted_by: number | null; +} + +/** The tool facts classification reads. */ +interface WriteToolFacts { + readonly method: string; + readonly path_template: string; +} + +/** The effect a state-changing tool has and its target: a state type, or a document tool by template. */ +export const classifyWrite = ( + tool: WriteToolFacts, + types: ReadonlySet, + documents: ReadonlyMap, +): { readonly effect: WriteEffect; readonly target: string | null } => { + const segments = segmentsOf(tool.path_template); + const path = normalizedPath(tool.path_template); + const parent = `/${segments.slice(0, -1).join("/")}`; + const item = segments.length > 0 && isParam(segments.at(-1)!); + if (tool.method === "POST" && types.has(path)) + return { effect: "create", target: path }; + if (item && types.has(parent)) { + if (tool.method === "DELETE") return { effect: "delete", target: parent }; + if (["PUT", "PATCH", "POST"].includes(tool.method)) + return { effect: "update", target: parent }; + } + const document = documents.get(path); + if (document !== undefined && ["PUT", "PATCH", "POST"].includes(tool.method)) + return { effect: "document", target: document }; + return { effect: "none", target: null }; +}; + +/** A plan for a tool whose effect is not modelled or cannot be built. */ +export const unservedPlan = ( + tool: string, + effect: WriteEffect, + target: string | null, + reason: WriteReason, +): StateWritePlan => ({ + tool, + effect, + target, + served: false, + reason, + exemplar_ordinal: null, + observation: null, + response: null, + body: null, + rules: [], + collection: null, + insert: null, + merge: null, + checked_record_ordinals: [], + contradicted_by: null, +}); + +/** One write call: its concrete path, the values its path parameters took and its body. */ +export interface WriteCall { + readonly path: string; + readonly pathValues: ReadonlyMap; + readonly body: JsonObject | undefined; +} + +/** The value each template parameter takes in a concrete path, decoded; undefined when the path does not instantiate it. */ +export const pathValuesOf = ( + template: string, + path: string, +): Map | undefined => { + const pattern = segmentsOf(template); + const concrete = segmentsOf(path); + if (pattern.length !== concrete.length) return undefined; + const values = new Map(); + for (const [index, segment] of pattern.entries()) { + const value = concrete[index]!; + if (!isParam(segment)) { + if (segment !== value) return undefined; + continue; + } + let decoded = value; + try { + decoded = decodeURIComponent(value); + } catch { + decoded = value; + } + values.set(segment.slice(1, -1), decoded); + } + return values; +}; + +/** Whether two paths are segment prefixes of one another, a parameter on either side matching any segment. */ +export const prefixRelated = (left: string, right: string): boolean => { + const a = segmentsOf(left); + const b = segmentsOf(right); + const shared = Math.min(a.length, b.length); + for (let index = 0; index < shared; index += 1) + if (!isParam(a[index]!) && !isParam(b[index]!) && a[index] !== b[index]) + return false; + return true; +}; diff --git a/src/domain/analysisView/environmentStateWriteTaint.ts b/src/domain/analysisView/environmentStateWriteTaint.ts new file mode 100644 index 0000000000..fbcbdce499 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteTaint.ts @@ -0,0 +1,108 @@ +import { + decodedSegment, + isParam, + normalizedPath, + segmentsOf, +} from "./environmentStateRules.js"; +import { + collectionTarget, + documentTarget, + entityTarget, + heldCollection, + itemTarget, + type SimulatedState, +} from "./environmentStateWriteChecks.js"; +import { prefixRelated } from "./environmentStateWriteRules.js"; + +/** + * What writes the gate does not model leave unknown: an entity below whose item path one wrote, + * or every entity of a collection when the path names none it holds (until a read shows each + * again); a collection one wrote at; everything under a path one wrote above collections; and + * targets whose check failed. + */ +export class WriteTaint { + readonly #prefixes: string[] = []; + readonly #targets = new Set(); + readonly #entities = new Set(); + /** Collections whose entities are all unknown, with the ones a read has shown since. */ + readonly #collections = new Map>(); + + constructor( + private readonly types: readonly string[], + private readonly documents: ReadonlyMap, + private readonly state: SimulatedState, + ) {} + + /** Record an unmodelled write's path against every related type and document, naming entities and collections as modelled writes do. */ + write(path: string): void { + const segments = segmentsOf(path); + for (const type of this.types) { + if (!prefixRelated(type, path)) continue; + const depth = segmentsOf(type).length; + const written = `/${segments.slice(0, depth).join("/")}`; + const collection = () => + collectionTarget(type, heldCollection(this.state, type, written)); + if (segments.length > depth) { + const item = itemTarget(this.state, type, { + path: `${written}/${segments[depth]!}`, + }); + if (this.state.entity(type, item.collection, item.id) !== undefined) + this.#entities.add(entityTarget(type, item.collection, item.id)); + else this.#collections.set(collection(), new Set()); + } else if (segments.length === depth) this.#targets.add(collection()); + else this.#prefixes.push(normalizedPath(path)); + } + for (const [tool, documentPath] of this.documents) + if (prefixRelated(documentPath, path)) + this.#targets.add(documentTarget(tool)); + } + + /** Stop checking a target whose check failed. */ + fail(target: string): void { + this.#targets.add(target); + } + + tainted(target: string): boolean { + return this.#targets.has(target); + } + + /** Whether an unmodelled write above a collection path left everything under it unknown, each collection segment matching the write's verbatim or once percent-decoded. */ + under(collection: string): boolean { + const concrete = segmentsOf(collection); + return this.#prefixes.some((prefix) => + segmentsOf(prefix).every( + (segment, index) => + isParam(segment) || + segment === concrete[index] || + decodedSegment(concrete[index] ?? "") === segment, + ), + ); + } + + unknown(type: string, collection: string, id: string): boolean { + const target = entityTarget(type, collection, id); + const seen = this.#collections.get(collectionTarget(type, collection)); + return ( + this.#entities.has(target) || (seen !== undefined && !seen.has(target)) + ); + } + + /** Whether a list read of a collection lists, or holds, an entity left unknown. */ + blocksList( + type: string, + collection: string, + ids: readonly string[], + ): boolean { + return ( + this.#collections.has(collectionTarget(type, collection)) || + ids.some((id) => this.#entities.has(entityTarget(type, collection, id))) + ); + } + + /** The state saw the entity again, or a modelled write replaced it. */ + known(type: string, collection: string, id: string): void { + const target = entityTarget(type, collection, id); + this.#entities.delete(target); + this.#collections.get(collectionTarget(type, collection))?.add(target); + } +} diff --git a/src/domain/analysisView/environmentStateWriteValues.test.ts b/src/domain/analysisView/environmentStateWriteValues.test.ts new file mode 100644 index 0000000000..acf0175a9b --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteValues.test.ts @@ -0,0 +1,256 @@ +import { expect, it } from "vitest"; +import type { JsonValue } from "../jsonValue.js"; +import { + containsDelimited, + delimitedOffsets, + formatClock, + integerOf, + isUuid, + readClock, + replaceDelimited, + seededToken, + seededUuid, + slugOf, + spelledLike, + type ClockFormat, +} from "./environmentStateWriteValues.js"; + +const WHOLE_SECOND = Date.UTC(2026, 2, 4, 5, 6, 7); +const WITH_MS = WHOLE_SECOND + 123; +const V4_UUID = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; +const UUID_EXEMPLAR = "123e4567-e89b-12d3-a456-426614174000"; + +const iso = ( + separator: "T" | " ", + fraction: number, + zone: string, +): ClockFormat => ({ kind: "iso", separator, fraction, zone }); + +const seeds = (count: number): string[] => + Array.from({ length: count }, (_, index) => `seed-${String(index)}`); + +it.each<[string, number, ClockFormat]>([ + ["2026-03-04T05:06:07Z", WHOLE_SECOND, iso("T", 0, "Z")], + ["2026-03-04 05:06:07.123Z", WITH_MS, iso(" ", 3, "Z")], + ["2026-03-04T05:06:07.123456Z", WITH_MS, iso("T", 6, "Z")], + ["2026-03-04T07:06:07.123+02:00", WITH_MS, iso("T", 3, "+02:00")], + ["2026-03-03T23:36:07-0530", WHOLE_SECOND, iso("T", 0, "-0530")], + ["2026-03-04T05:06:07", WHOLE_SECOND, iso("T", 0, "")], + ["2024-02-29 00:00:00Z", Date.UTC(2024, 1, 29), iso(" ", 0, "Z")], +])("reads the ISO clock %s with its spelling", (text, ms, format) => { + expect(readClock(text)).toEqual({ ms, format }); +}); + +it.each([ + "2026-02-30T00:00:00Z", + "2026-02-29T00:00:00Z", + "2026-13-01T00:00:00Z", + "2026-03-04T24:00:00Z", + "2026-03-04T05:60:00Z", + "2026-03-04T05:06:60Z", + "2026-03-04", + "2026-03-04T05:06Z", + "2026-03-04T05:06:07+05", + "2026-03-04T05:06:07.1234567890Z", + "2026-03-04t05:06:07Z", + "next tuesday", +])("refuses %s as a clock", (text) => { + expect(readClock(text)).toBeUndefined(); +}); + +it.each<[JsonValue | undefined, ReturnType]>([ + [ + WHOLE_SECOND / 1000, + { ms: WHOLE_SECOND, format: { kind: "epoch", unit: "s" } }, + ], + [WITH_MS, { ms: WITH_MS, format: { kind: "epoch", unit: "ms" } }], + [999_999_999, undefined], + [99_999_999_999, undefined], + [10_000_000_000_000, undefined], + [WHOLE_SECOND / 1000 + 0.5, undefined], + [String(WHOLE_SECOND / 1000), undefined], + [null, undefined], + [undefined, undefined], +])("reads the epoch value %j", (value, expected) => { + expect(readClock(value)).toEqual(expected); +}); + +it.each([ + "2026-03-04T05:06:07Z", + "2026-03-04 05:06:07.123Z", + "2026-03-04T07:06:07.123+02:00", + "2026-03-03T23:36:07-0530", + "2026-03-04T05:06:07.1", + "2026-03-04T05:06:07.123000000", + WHOLE_SECOND / 1000, + WITH_MS, +])("formats %j back to its own spelling", (value) => { + const clock = readClock(value)!; + expect(formatClock(clock.ms, clock.format)).toBe(value); +}); + +it.each<[ClockFormat, JsonValue]>([ + [iso("T", 6, "Z"), "2026-03-04T05:06:07.123000Z"], + [iso(" ", 9, ""), "2026-03-04 05:06:07.123000000"], + [iso("T", 1, "Z"), "2026-03-04T05:06:07.1Z"], + [iso("T", 0, "+02:00"), "2026-03-04T07:06:07+02:00"], + [iso("T", 3, "-0530"), "2026-03-03T23:36:07.123-0530"], + [{ kind: "epoch", unit: "s" }, WHOLE_SECOND / 1000], + [{ kind: "epoch", unit: "ms" }, WITH_MS], +])("formats an instant as %j", (format, expected) => { + expect(formatClock(WITH_MS, format)).toBe(expected); +}); + +it.each<[JsonValue | undefined, boolean]>([ + [UUID_EXEMPLAR, true], + [UUID_EXEMPLAR.toUpperCase(), true], + [UUID_EXEMPLAR.replaceAll("-", ""), false], + [UUID_EXEMPLAR.slice(1), false], + [`g${UUID_EXEMPLAR.slice(1)}`, false], + [` ${UUID_EXEMPLAR}`, false], + [42, false], + [undefined, false], +])("isUuid(%j) is %s", (value, expected) => { + expect(isUuid(value)).toBe(expected); +}); + +it.each<[JsonValue | undefined, number | undefined]>([ + [42, 42], + [-3, -3], + ["42", 42], + ["007", 7], + ["123456789012345", 123_456_789_012_345], + ["1234567890123456", undefined], + ["-3", undefined], + ["4.2", undefined], + [4.2, undefined], + [2 ** 53, undefined], + ["", undefined], + [" 4", undefined], + [true, undefined], + [null, undefined], + [undefined, undefined], +])("integerOf(%j) is %j", (value, expected) => { + expect(integerOf(value)).toBe(expected); +}); + +it("spells an integer as the recorded value spells integers", () => { + expect(spelledLike("7", 8)).toBe("8"); + expect(spelledLike(7, 8)).toBe(8); + expect(spelledLike(null, 8)).toBe(8); +}); + +it.each([ + ["Hello World", "hello-world"], + ["Héllo, Wörld!", "hello-world"], + [" Crème brûlée -- recipe__v2 ", "creme-brulee-recipe-v2"], + ["Ｆｕｌｌｗｉｄｔｈ ２", "fullwidth-2"], + ["already-a-slug", "already-a-slug"], + ["--Edge--", "edge"], + ["!!!", ""], +])("slugOf(%j) is %j", (text, slug) => { + expect(slugOf(text)).toBe(slug); +}); + +it.each<[string, string, number[]]>([ + ["/issues/2", "2", [8]], + ["2026", "2", []], + ["2 of 22, page 2", "2", [0, 14]], + ["x2/2y", "2", []], + ["é2", "2", []], + ["https://x.test/items/abc123?ref=abc123", "abc123", [21, 32]], + ["anything", "", []], +])("finds %j's delimited %j at %j", (text, token, offsets) => { + expect(delimitedOffsets(text, token)).toEqual(offsets); +}); + +it("replaces only delimited occurrences", () => { + expect(replaceDelimited("/issues/2 in 2026", "2", "9")).toBe( + "/issues/9 in 2026", + ); + expect(replaceDelimited("a-2-b-2", "2", "10")).toBe("a-10-b-10"); + expect(replaceDelimited("2026", "2", "9")).toBe("2026"); +}); + +// Known bug: a rejected match skips its whole length, hiding an overlapping delimited one. +it("finds a delimited match that overlaps an undelimited one", () => { + expect(delimitedOffsets("11-1-1", "1-1")).toEqual([3]); +}); + +it("finds a delimited token in any string nested in objects and arrays", () => { + expect( + containsDelimited({ a: [1, { b: ["x", "see /issues/2"] }] }, "2"), + ).toBe(true); + expect(containsDelimited("2", "2")).toBe(true); + expect(containsDelimited({ c: "2026", n: 2, d: null }, "2")).toBe(false); + expect(containsDelimited(["2026", true, [{ e: "x2" }]], "2")).toBe(false); +}); + +it("searches arrays and objects too wide to spread into one call", () => { + const wide = Array.from({ length: 500_000 }, () => 0); + expect(containsDelimited(wide, "2")).toBe(false); + expect(containsDelimited({ wide: [...wide, "see /issues/2"] }, "2")).toBe( + true, + ); + expect( + containsDelimited( + Object.fromEntries(wide.map((value, index) => [String(index), value])), + "2", + ), + ).toBe(false); +}); + +it("derives a deterministic version 4 UUID per seed", () => { + const first = seededUuid("seed-a", UUID_EXEMPLAR); + expect(first).toMatch(V4_UUID); + expect(seededUuid("seed-a", UUID_EXEMPLAR)).toBe(first); + const derived = seeds(64).map((seed) => seededUuid(seed, UUID_EXEMPLAR)); + expect(new Set(derived).size).toBe(derived.length); + for (const uuid of derived) expect(uuid).toMatch(V4_UUID); +}); + +it("keeps upper case only when the recorded UUID is upper case", () => { + const upper = seededUuid("seed-a", UUID_EXEMPLAR.toUpperCase()); + expect(upper).toBe(seededUuid("seed-a", UUID_EXEMPLAR).toUpperCase()); + expect(upper).toMatch(/[A-F]/u); + const digits = "12345678-1234-4234-8234-123456789012"; + expect(seededUuid("seed-a", digits)).toMatch(V4_UUID); + const mixed = "123e4567-E89B-12d3-a456-426614174000"; + expect(seededUuid("seed-a", mixed)).toMatch(V4_UUID); +}); + +it.each<[string, RegExp]>([ + ["a1b2c3d4e5f6", /^[0-9a-f]{12}$/u], + ["A1B2C3D4E5F6", /^[0-9A-F]{12}$/u], + ["k9x2m7q4z1w8", /^[0-9a-z]{12}$/u], + ["AbCdEfGhIjKl", /^[A-Za-z]{12}$/u], + ["deadbeefcafe", /^[a-z]{12}$/u], + [ + "tok_9Xa-7bQ.2", + /^[0-9A-Za-z]{3}_[0-9A-Za-z]{3}-[0-9A-Za-z]{3}\.[0-9A-Za-z]$/u, + ], + ["____", /^____$/u], +])("keeps the length and character classes of %j", (exemplar, shape) => { + for (const seed of seeds(16)) { + const token = seededToken(seed, exemplar); + expect(token).toHaveLength(exemplar.length); + expect(token).toMatch(shape); + expect(seededToken(seed, exemplar)).toBe(token); + } +}); + +it("draws tokens from the whole alphabet the recorded token uses", () => { + const characters = (exemplar: string): string => + seeds(64) + .map((seed) => seededToken(seed, exemplar)) + .join(""); + expect(characters("a1b2c3d4e5f6")).toMatch(/^[0-9a-f]+$/u); + expect(characters("k9x2m7q4z1w8")).toMatch(/[g-z]/u); + const mixed = characters("AbCdEfGhIjKl"); + expect(mixed).toMatch(/[a-z]/u); + expect(mixed).toMatch(/[A-Z]/u); + expect(seededToken("seed-a", "k9x2m7q4z1w8")).not.toBe( + seededToken("seed-b", "k9x2m7q4z1w8"), + ); +}); diff --git a/src/domain/analysisView/environmentStateWriteValues.ts b/src/domain/analysisView/environmentStateWriteValues.ts new file mode 100644 index 0000000000..10e814f22a --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteValues.ts @@ -0,0 +1,236 @@ +import { createHash } from "node:crypto"; +import type { JsonValue } from "../jsonValue.js"; + +/** How a recorded clock value is spelled: ISO-8601 with its separator, fraction digits and zone, or an epoch number. */ +export type ClockFormat = + | { + readonly kind: "iso"; + readonly separator: "T" | " "; + readonly fraction: number; + readonly zone: string; + } + | { readonly kind: "epoch"; readonly unit: "s" | "ms" }; + +const ISO_INSTANT = + /^(\d{4})-(\d{2})-(\d{2})([T ])(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:?\d{2})?$/u; +const MINUTE_MS = 60_000; + +/** Minutes east of UTC that a zone suffix names; none and `Z` are UTC. */ +const zoneMinutes = (zone: string): number => { + if (zone === "" || zone === "Z") return 0; + const digits = zone.slice(1).replace(":", ""); + const minutes = + Number(digits.slice(0, 2)) * 60 + Number(digits.slice(2, 4) || "0"); + return zone.startsWith("-") ? -minutes : minutes; +}; + +/** The instant of an ISO-8601 date-time and how it is spelled, or undefined when it is not one. */ +const readIsoClock = ( + value: string, +): { readonly ms: number; readonly format: ClockFormat } | undefined => { + const match = ISO_INSTANT.exec(value); + if (match === null) return undefined; + const [year, month, day, separator, hour, minute, second] = match + .slice(1, 8) + .map((part) => (part === "T" || part === " " ? part : Number(part))); + const fraction = match[8] ?? ""; + const zone = match[9] ?? ""; + const local = Date.UTC( + year as number, + (month as number) - 1, + day as number, + hour as number, + minute as number, + second as number, + Number(`${fraction}000`.slice(0, 3)), + ); + // Date.UTC rolls invalid fields over; the round trip refuses them. + const rolled = new Date(local); + if ( + rolled.getUTCFullYear() !== year || + rolled.getUTCMonth() !== (month as number) - 1 || + rolled.getUTCDate() !== day || + rolled.getUTCHours() !== hour || + rolled.getUTCMinutes() !== minute + ) + return undefined; + return { + ms: local - zoneMinutes(zone) * MINUTE_MS, + format: { + kind: "iso", + separator: separator as "T" | " ", + fraction: fraction.length, + zone, + }, + }; +}; + +/** The instant a clock value names and its format: an ISO-8601 date-time, or epoch seconds or milliseconds. */ +export const readClock = ( + value: JsonValue | undefined, +): { readonly ms: number; readonly format: ClockFormat } | undefined => { + if (typeof value === "string") return readIsoClock(value); + if (typeof value !== "number" || !Number.isSafeInteger(value)) + return undefined; + if (value >= 1e9 && value < 1e10) + return { ms: value * 1000, format: { kind: "epoch", unit: "s" } }; + if (value >= 1e12 && value < 1e13) + return { ms: value, format: { kind: "epoch", unit: "ms" } }; + return undefined; +}; + +const pad = (value: number, width: number): string => + String(value).padStart(width, "0"); + +/** An instant spelled in a recorded clock format; fraction digits past milliseconds are zeros. */ +export const formatClock = (ms: number, format: ClockFormat): JsonValue => { + if (format.kind === "epoch") + return format.unit === "s" ? Math.floor(ms / 1000) : ms; + const at = new Date(ms + zoneMinutes(format.zone) * MINUTE_MS); + const fraction = + format.fraction === 0 + ? "" + : `.${`${pad(at.getUTCMilliseconds(), 3)}${"0".repeat(9)}`.slice(0, format.fraction)}`; + return `${pad(at.getUTCFullYear(), 4)}-${pad(at.getUTCMonth() + 1, 2)}-${pad(at.getUTCDate(), 2)}${format.separator}${pad(at.getUTCHours(), 2)}:${pad(at.getUTCMinutes(), 2)}:${pad(at.getUTCSeconds(), 2)}${fraction}${format.zone}`; +}; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu; + +export const isUuid = (value: JsonValue | undefined): value is string => + typeof value === "string" && UUID.test(value); + +/** An integer value as a number, from a JSON number or a string of digits. */ +export const integerOf = (value: JsonValue | undefined): number | undefined => + typeof value === "number" && Number.isSafeInteger(value) + ? value + : typeof value === "string" && /^\d{1,15}$/u.test(value) + ? Number(value) + : undefined; + +/** + * The next value of a sequence. With nothing taken yet, that is the next held integer, or 1 when + * the scope holds none. Otherwise it is one past the later of that and the last value taken. + * A missing side leaves a negative value negative. + */ +export const nextSequence = ( + held: number | undefined, + taken: number | undefined, +): number => + taken === undefined ? (held ?? 1) : Math.max(held ?? taken + 1, taken + 1); + +/** An integer spelled as the recorded value spells integers: a number, or a string of digits. */ +export const spelledLike = (exemplar: JsonValue, value: number): JsonValue => + typeof exemplar === "string" ? String(value) : value; + +/** Lower case, with every run of other characters joined by one `-` and none at either end. */ +export const slugOf = (text: string): string => + text + .normalize("NFKD") + .replace(/[̀-ͯ]/gu, "") + .toLowerCase() + .replace(/[^a-z0-9]+/gu, "-") + .replace(/^-+|-+$/gu, ""); + +const isAlphanumeric = (character: string | undefined): boolean => + character !== undefined && /[\p{L}\p{N}]/u.test(character); + +/** Start offsets of `token` in `text` where neither neighbour is a letter or digit. */ +export const delimitedOffsets = (text: string, token: string): number[] => { + const offsets: number[] = []; + if (token.length === 0) return offsets; + let at = text.indexOf(token); + while (at >= 0) { + const delimited = + !isAlphanumeric(text[at - 1]) && !isAlphanumeric(text[at + token.length]); + if (delimited) offsets.push(at); + // A rejected match may overlap a delimited one, so the search moves on by one character. + at = text.indexOf(token, at + (delimited ? token.length : 1)); + } + return offsets; +}; + +/** `text` with every delimited occurrence of `token` replaced. */ +export const replaceDelimited = ( + text: string, + token: string, + replacement: string, +): string => { + let result = ""; + let from = 0; + for (const at of delimitedOffsets(text, token)) { + result += text.slice(from, at) + replacement; + from = at + token.length; + } + return result + text.slice(from); +}; + +/** Whether any string inside a value holds `token` delimited. */ +export const containsDelimited = (value: JsonValue, token: string): boolean => { + const pending: JsonValue[] = [value]; + for (let next = pending.pop(); next !== undefined; next = pending.pop()) { + if (typeof next === "string") { + if (delimitedOffsets(next, token).length > 0) return true; + } else if (typeof next === "object" && next !== null) + // One at a time: spreading a wide array into push overflows the call stack. + for (const child of Array.isArray(next) ? next : Object.values(next)) + pending.push(child); + } + return false; +}; + +/** SHA-256 bytes of a seed, extended by counter blocks to the requested length. */ +const seededBytes = (seed: string, length: number): Uint8Array => { + const bytes = new Uint8Array(length); + for (let block = 0; block * 32 < length; block += 1) { + const digest = createHash("sha256") + .update(`${seed}#${String(block)}`) + .digest(); + bytes.set( + digest.subarray(0, Math.min(32, length - block * 32)), + block * 32, + ); + } + return bytes; +}; + +/** A version 4 UUID derived from a seed, in the recorded value's letter case. */ +export const seededUuid = (seed: string, exemplar: string): string => { + const bytes = seededBytes(seed, 16); + bytes[6] = (bytes[6]! & 0x0f) | 0x40; + bytes[8] = (bytes[8]! & 0x3f) | 0x80; + const hex = Buffer.from(bytes).toString("hex"); + const uuid = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; + return exemplar === exemplar.toUpperCase() && /[A-F]/u.test(exemplar) + ? uuid.toUpperCase() + : uuid; +}; + +const DIGITS = "0123456789"; +const LOWER = "abcdefghijklmnopqrstuvwxyz"; +const UPPER = LOWER.toUpperCase(); + +/** The characters a token of this spelling draws from: hexadecimal, or the digit and letter cases it uses. */ +const tokenAlphabet = (exemplar: string): string => { + if (/^[0-9a-f]+$/u.test(exemplar) && /[0-9]/u.test(exemplar)) + return `${DIGITS}abcdef`; + if (/^[0-9A-F]+$/u.test(exemplar) && /[0-9]/u.test(exemplar)) + return `${DIGITS}ABCDEF`; + const alphabet = + (/[0-9]/u.test(exemplar) ? DIGITS : "") + + (/[a-z]/u.test(exemplar) ? LOWER : "") + + (/[A-Z]/u.test(exemplar) ? UPPER : ""); + return alphabet === "" ? LOWER : alphabet; +}; + +/** A token as long as the recorded one, from its alphabet, keeping its other characters in place. */ +export const seededToken = (seed: string, exemplar: string): string => { + const alphabet = tokenAlphabet(exemplar); + const bytes = seededBytes(seed, exemplar.length); + return [...exemplar] + .map((character, index) => + /[0-9A-Za-z]/u.test(character) + ? alphabet[bytes[index]! % alphabet.length]! + : character, + ) + .join(""); +}; diff --git a/src/domain/analysisView/environmentStateWriteVerdict.ts b/src/domain/analysisView/environmentStateWriteVerdict.ts new file mode 100644 index 0000000000..d0d7c1fb61 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteVerdict.ts @@ -0,0 +1,59 @@ +import { at, isObject, type StateEntity } from "./environmentStateRules.js"; +import type { WriteDraft, WriteIo } from "./environmentStateWriteEffects.js"; +import type { StateWritePlan } from "./environmentStateWriteRules.js"; + +/** Whether an update or delete that answers with the entity can answer for an initial entity: one holds, or an update's rules write, every field its recorded response shows. */ +const renders = ( + plan: StateWritePlan, + input: { readonly io: WriteIo; readonly entities: readonly StateEntity[] }, +): boolean => { + const { response } = plan; + if ( + (plan.effect !== "update" && plan.effect !== "delete") || + response?.shape !== "entity" + ) + return true; + const shown = input.io.withBody(response.ordinal, (body) => { + const object = + body === undefined ? undefined : at(body, response.pointer ?? ""); + return isObject(object) ? Object.keys(object) : undefined; + }); + // Serving renders an update's answer from the entity it wrote; an increment only adds to a field held. + const written = new Set( + plan.effect === "update" + ? plan.rules.flatMap((rule) => + rule.rule === "increment" ? [] : [rule.field], + ) + : [], + ); + return ( + shown === undefined || + input.entities.some( + (entity) => + entity.type === plan.target && + shown.every( + (field) => Object.hasOwn(entity.fields, field) || written.has(field), + ), + ) + ); +}; + +/** A replayed write's plan, served only when it was called, nothing contradicted it, a read confirmed it and its answers can render. */ +export const writeVerdict = ( + draft: WriteDraft, + input: { readonly io: WriteIo; readonly entities: readonly StateEntity[] }, +): StateWritePlan => { + const plan = draft.plan; + if (plan.reason === null) { + if (draft.calls === 0) plan.reason = "no-call"; + else if (draft.contradictedBy !== undefined) { + plan.reason = "contradicted"; + plan.contradicted_by = draft.contradictedBy; + } else if (draft.sideEffect) plan.reason = "side-effect"; + else if (draft.confirmed.length === 0) plan.reason = "unconfirmed"; + else if (!renders(plan, input)) plan.reason = "incomplete-entities"; + } + plan.served = plan.reason === null; + plan.checked_record_ordinals = [...new Set(draft.confirmed)]; + return { ...plan }; +}; diff --git a/src/domain/analysisView/environmentStateWrites.test.ts b/src/domain/analysisView/environmentStateWrites.test.ts new file mode 100644 index 0000000000..e813071ae2 --- /dev/null +++ b/src/domain/analysisView/environmentStateWrites.test.ts @@ -0,0 +1,543 @@ +import { describe, expect, it } from "vitest"; + +import type { JsonValue } from "../jsonValue.js"; +import { + timedCaptureEvidence, + type TimedCaptureRow, +} from "../../../tests/fixtures/analysisView.js"; +import { + BOOK_INIT_ROWS, + BOOK_NOT_FOUND, + BOOK_ROWS, + BOOK_WRITE_CLOCK, + FIRST_BOOK, + SECOND_BOOK, + THIRD_BOOK, + book, + bookClock, + environmentStateViewOf, + rawCaptureEvidence, + renamedBook, + type Book, +} from "../../../tests/fixtures/environmentStateWrites.js"; + +type StateView = ReturnType; + +const ISO_SECONDS = { + kind: "iso", + separator: "T", + fraction: 0, + zone: "Z", +} as const; + +const stateOf = (rows: readonly TimedCaptureRow[]): StateView => + environmentStateViewOf(timedCaptureEvidence(rows)); + +const planOf = (state: StateView, tool: string) => { + const plan = state.writes.find((candidate) => candidate.tool === tool); + if (plan === undefined) throw new Error(`No write plan for ${tool}`); + return plan; +}; + +/** Each write tool's verdict: "served", else the reason it is not. */ +const verdicts = (rows: readonly TimedCaptureRow[]) => + Object.fromEntries( + stateOf(rows).writes.map((plan) => [ + plan.tool, + plan.served ? "served" : plan.reason, + ]), + ); + +/** A create of the third book whose response is `response`, then the given later exchanges. */ +const created = ( + response: JsonValue, + ...later: TimedCaptureRow[] +): TimedCaptureRow[] => [ + ...BOOK_INIT_ROWS, + ["POST", "/api/books", response, 201, { name: "Third Book" }], + ...later, +]; + +const RENAMED_AT_3 = renamedBook(FIRST_BOOK, "Renamed Book", 3); +/** An update renaming the first book whose response is `response`, then the given later exchanges. */ +const rename = ( + response: Book = RENAMED_AT_3, + ...later: TimedCaptureRow[] +): TimedCaptureRow[] => [ + ...BOOK_INIT_ROWS, + ["PATCH", "/api/books/1", response, 200, { name: "Renamed Book" }], + ...later, +]; + +describe("served write plans", () => { + it("serves the create, update and delete that later reads confirm, citing those reads", () => { + const state = stateOf(BOOK_ROWS); + expect(state.write_clock).toBe(BOOK_WRITE_CLOCK); + expect(planOf(state, "post_books")).toEqual({ + tool: "post_books", + effect: "create", + target: "/api/books", + served: true, + reason: null, + exemplar_ordinal: 3, + observation: { ordinal: 3, pointer: "" }, + response: { ordinal: 3, shape: "entity", pointer: "" }, + body: { + accepted: ["name"], + fixed: [], + required: ["name"], + identity: null, + kinds: [], + }, + rules: [ + { field: "id", rule: "identity", generator: "sequence" }, + { field: "name", rule: "echo" }, + { field: "slug", rule: "slug", from: "name" }, + { field: "created_at", rule: "clock", format: ISO_SECONDS }, + { field: "updated_at", rule: "clock", format: ISO_SECONDS }, + ], + collection: null, + insert: { rule: "sorted", field: "id", direction: "ascending" }, + merge: null, + // The final list follows the delete of book 3, so it shows nothing of the create. + checked_record_ordinals: [4], + contradicted_by: null, + }); + // PATCH requires no member; the slug follows the sent name and the update stamps its clock. + expect(planOf(state, "patch_books_by_id")).toMatchObject({ + effect: "update", + served: true, + exemplar_ordinal: 5, + body: { accepted: ["name"], required: [] }, + rules: [ + { field: "name", rule: "echo" }, + { field: "slug", rule: "slug", from: "name" }, + { field: "updated_at", rule: "clock", format: ISO_SECONDS }, + ], + checked_record_ordinals: [6, 9], + }); + expect(planOf(state, "delete_books_by_id")).toMatchObject({ + effect: "delete", + served: true, + exemplar_ordinal: 7, + response: { ordinal: 7, shape: "empty", pointer: null }, + checked_record_ordinals: [8, 9], + }); + expect(planOf(state, "post_exports")).toMatchObject({ + effect: "none", + target: null, + served: false, + reason: "no-state-target", + }); + }); + + it("starts the clock from the records a date can hold when one lies past its range", () => { + const state = stateOf([ + ["GET", "/api/books", [FIRST_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + [ + "PATCH", + "/api/books/1", + RENAMED_AT_3, + 200, + { name: "Renamed Book" }, + 9_000_000_000_000_000, + ], + ]); + // The second read ends at 18:00:01.005, so the clock starts at the next whole second. + expect(state.write_clock).toBe("2026-10-17T18:00:02.000Z"); + }); + + it("starts no clock on a record-order timeline, where no value is generated by a clock", () => { + const state = environmentStateViewOf( + rawCaptureEvidence( + created(THIRD_BOOK, ["GET", "/api/books/3", THIRD_BOOK]), + { + mimeType: "application/json", + timed: false, + }, + ), + ); + expect(state.timeline).toBe("record-order"); + expect(state.write_clock).toBeNull(); + const plan = planOf(state, "post_books"); + expect(plan).toMatchObject({ served: true, checked_record_ordinals: [4] }); + // Without start times the clock fields keep the recorded values as constants. + expect(plan.rules.map(({ field }) => field)).toEqual([ + "id", + "name", + "slug", + ]); + }); +}); + +describe("unserved write plans", () => { + it("names a write to no state type and a tool with no successful call", () => { + expect( + verdicts([ + ...BOOK_INIT_ROWS, + ["POST", "/api/session", { token: "t0" }, 200, { user: "ada" }], + ["POST", "/api/books", { errors: ["name"] }, 422, { name: "" }], + ["DELETE", "/api/books/9", BOOK_NOT_FOUND, 404], + ]), + ).toEqual({ + post_session: "no-state-target", + post_books: "no-call", + delete_books_by_id: "no-call", + }); + }); + + it("refuses a create whose request body cannot enter the state", () => { + /** The create of the third book sending `sent`, or no body, then a read that shows it. */ + const sending = (...sent: [] | [JsonValue]): TimedCaptureRow[] => [ + ...BOOK_INIT_ROWS, + ["POST", "/api/books", THIRD_BOOK, 201, ...sent], + ["GET", "/api/books/3", THIRD_BOOK], + ]; + const recorded = (mimeType: string, text: string) => + environmentStateViewOf( + rawCaptureEvidence(sending(text), { mimeType, timed: true }), + ); + for (const state of [ + stateOf(sending()), + stateOf(sending([{ name: "Third Book" }])), + recorded("application/x-www-form-urlencoded", "name=Third+Book"), + recorded("application/json", '{"__proto__":{},"name":"Third Book"}'), + recorded( + "application/json", + '{"name":"Third Book","copies":12345678901234567890}', + ), + ]) + expect(planOf(state, "post_books")).toMatchObject({ + served: false, + reason: "body-unusable", + exemplar_ordinal: null, + }); + }); + + it("reports no observation when neither the response nor a read before the next write shows the entity", () => { + expect(verdicts(created({ ok: true }))).toEqual({ + post_books: "no-observation", + }); + // The read of the created book comes after the next write; the update names a book never held. + expect( + verdicts( + created( + "", + [ + "PATCH", + "/api/books/7", + { id: 7, name: "Seventh" }, + 200, + { name: "Seventh" }, + ], + ["GET", "/api/books/3", THIRD_BOOK], + ), + ), + ).toEqual({ + post_books: "no-observation", + patch_books_by_id: "no-observation", + }); + }); + + it("reports an update that changed an unsent field no clock, increment or slug explains", () => { + const changed = { ...RENAMED_AT_3, slug: "first-book-v2" }; + expect(verdicts(rename(changed, ["GET", "/api/books/1", changed]))).toEqual( + { + patch_books_by_id: "unexplained-change", + }, + ); + }); + + it("names the first later read that contradicts a write", () => { + const other = renamedBook(RENAMED_AT_3, "Other Name", 3); + const plan = planOf( + stateOf( + rename( + RENAMED_AT_3, + ["GET", "/api/books/1", other], + ["GET", "/api/books/1", other], + ), + ), + "patch_books_by_id", + ); + expect(plan).toMatchObject({ + served: false, + reason: "contradicted", + contradicted_by: 4, + checked_record_ordinals: [], + }); + }); + + it("blames a related write when a book no write touched changed after it", () => { + const plan = planOf( + stateOf( + created( + THIRD_BOOK, + ["GET", "/api/books/3", THIRD_BOOK], + [ + "GET", + "/api/books/2", + renamedBook(SECOND_BOOK, "Second Edition", 5), + ], + ), + ), + "post_books", + ); + // The create's own read confirmed it, but the changed sibling outweighs that. + expect(plan).toMatchObject({ + served: false, + reason: "side-effect", + checked_record_ordinals: [4], + contradicted_by: null, + }); + }); + + it("leaves a write that no later read checks unconfirmed", () => { + expect(planOf(stateOf(created(THIRD_BOOK)), "post_books")).toMatchObject({ + served: false, + reason: "unconfirmed", + exemplar_ordinal: 3, + checked_record_ordinals: [], + }); + }); +}); + +describe("delete confirmation", () => { + it("confirms a delete by a later 404, 400 or JSON null item read", () => { + const absences: [body: JsonValue, status: number][] = [ + [BOOK_NOT_FOUND, 404], + [{ detail: "Invalid book id." }, 400], + [null, 200], + ]; + for (const [body, status] of absences) + expect( + planOf( + stateOf([ + ...BOOK_INIT_ROWS, + ["DELETE", "/api/books/2", "", 204], + ["GET", "/api/books/2", body, status], + ]), + "delete_books_by_id", + ), + ).toMatchObject({ served: true, checked_record_ordinals: [4] }); + }); + it("withholds a delete whose recorded answer shows fields no initial entity holds", () => { + const answered = (previous: JsonValue) => + planOf( + stateOf([ + ...BOOK_INIT_ROWS, + ["DELETE", "/api/books/2", { deleted: true, previous }, 200], + ["GET", "/api/books/2", BOOK_NOT_FOUND, 404], + ]), + "delete_books_by_id", + ); + // An edit view of the book: the read views never showed its draft field. + expect(answered({ ...SECOND_BOOK, draft: false })).toMatchObject({ + served: false, + reason: "incomplete-entities", + checked_record_ordinals: [4], + }); + expect(answered(SECOND_BOOK)).toMatchObject({ + served: true, + response: { shape: "entity", pointer: "/previous" }, + }); + }); +}); + +describe("update answers", () => { + it("serves an update whose recorded answer shows a field only the update writes", () => { + const summary = (name: string) => [ + { id: 1, name }, + { id: 2, name: "B" }, + ]; + // No read showed a note before the update sent one, which its answer echoes. + const plan = planOf( + stateOf([ + ["GET", "/api/books", summary("A")], + ["GET", "/api/books/1", { id: 1, name: "A" }], + ["GET", "/api/books/2", { id: 2, name: "B" }], + [ + "PATCH", + "/api/books/1", + { id: 1, name: "Z", note: "n" }, + 200, + { name: "Z", note: "n" }, + ], + ["GET", "/api/books", summary("Z")], + ]), + "patch_books_by_id", + ); + expect(plan).toMatchObject({ + served: true, + reason: null, + response: { shape: "entity", pointer: "" }, + checked_record_ordinals: [4], + }); + }); +}); + +describe("insert rules", () => { + it("learns where a create inserts from the recorded collection order", () => { + // Listed by name, while neither identities nor clocks are in order. + const shelf = [ + book(3, "Alpha", "2026-10-01T10:00:00Z"), + book(1, "Delta", "2026-10-01T09:00:00Z"), + book(2, "Zulu", "2026-10-01T11:00:00Z"), + ]; + const mike = book(4, "Mike", bookClock(6)); + const state = stateOf([ + ["GET", "/api/books", shelf], + ...shelf.map((entry): TimedCaptureRow => [ + "GET", + `/api/books/${String(entry.id)}`, + entry, + ]), + ["GET", "/api/books", shelf], + ["GET", "/api/books", shelf], + ["POST", "/api/books", mike, 201, { name: "Mike" }], + ["GET", "/api/books", [shelf[0]!, shelf[1]!, mike, shelf[2]!]], + ]); + expect(planOf(state, "post_books")).toMatchObject({ + served: true, + insert: { rule: "sorted", field: "name", direction: "ascending" }, + checked_record_ordinals: [7], + }); + }); + + it("takes the position a later list shows when the recorded order admits several", () => { + const zed = book(3, "Zed", bookClock(3)); + const plan = planOf( + stateOf( + created(zed, ["GET", "/api/books", [zed, FIRST_BOOK, SECOND_BOOK]]), + ), + "post_books", + ); + expect(plan).toMatchObject({ + served: true, + insert: { rule: "first" }, + checked_record_ordinals: [4], + }); + }); +}); + +describe("unmodelled writes", () => { + const shelved = (entry: Book, archived = false): Book => ({ + ...entry, + archived, + }); + /** The rename, an archive the gate does not model when `archive` is set, then a read of both changes. */ + const archivedAfterRename = (archive: boolean) => + stateOf([ + ["GET", "/api/books", [shelved(FIRST_BOOK), shelved(SECOND_BOOK)]], + ["GET", "/api/books/1", shelved(FIRST_BOOK)], + ["GET", "/api/books/2", shelved(SECOND_BOOK)], + [ + "PATCH", + "/api/books/1", + shelved(RENAMED_AT_3), + 200, + { name: "Renamed Book" }, + ], + ...(archive + ? [ + [ + "POST", + "/api/books/1/archive", + { archived: true }, + 200, + {}, + ] as TimedCaptureRow, + ] + : []), + ["GET", "/api/books/1", shelved(RENAMED_AT_3, true)], + ]); + + it("confirms an update on the members it sent when an unmodelled write also changed the entity", () => { + const state = archivedAfterRename(true); + expect(planOf(state, "patch_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [5], + contradicted_by: null, + }); + expect(planOf(state, "post_books_archive_by_id")).toMatchObject({ + effect: "none", + reason: "no-state-target", + }); + // With no unmodelled write the same read contradicts the update. + expect( + planOf(archivedAfterRename(false), "patch_books_by_id"), + ).toMatchObject({ + reason: "contradicted", + contradicted_by: 4, + }); + }); + + it("does not blame a modelled write for a check that fails after an unmodelled write", () => { + const changedSibling = renamedBook(SECOND_BOOK, "Second Edition", 4); + const reindex: TimedCaptureRow = [ + "POST", + "/api/jobs/reindex", + { queued: true }, + 202, + {}, + ]; + const list: TimedCaptureRow = [ + "GET", + "/api/books", + [RENAMED_AT_3, changedSibling], + ]; + expect( + planOf(stateOf(rename(RENAMED_AT_3, list)), "patch_books_by_id"), + ).toMatchObject({ + reason: "contradicted", + contradicted_by: 4, + }); + expect( + planOf(stateOf(rename(RENAMED_AT_3, reindex, list)), "patch_books_by_id"), + ).toMatchObject({ reason: "unconfirmed", contradicted_by: null }); + // A create wrote every field, so after an unmodelled write a differing read checks none of them. + const reslugged = { ...THIRD_BOOK, slug: "third-book-1" }; + const read: TimedCaptureRow = ["GET", "/api/books/3", reslugged]; + expect( + planOf(stateOf(created(THIRD_BOOK, read)), "post_books"), + ).toMatchObject({ + reason: "contradicted", + contradicted_by: 4, + }); + expect( + planOf(stateOf(created(THIRD_BOOK, reindex, read)), "post_books"), + ).toMatchObject({ + reason: "unconfirmed", + contradicted_by: null, + }); + }); +}); + +describe("observation from a later read", () => { + it("observes a created entity in the first later item read when the response names only its identity", () => { + const later: TimedCaptureRow[] = [ + ["GET", "/api/books/3", THIRD_BOOK], + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK, THIRD_BOOK]], + ]; + const bare = planOf(stateOf(created({ id: 3 }, ...later)), "post_books"); + // The read that showed the entity does not check the create; the later list does. + expect(bare).toMatchObject({ + served: true, + exemplar_ordinal: 3, + observation: { ordinal: 4, pointer: "" }, + response: { ordinal: 3, shape: "entity", pointer: "" }, + checked_record_ordinals: [5], + }); + const named = planOf( + stateOf(created({ created_id: 3 }, ...later)), + "post_books", + ); + expect(named).toMatchObject({ + served: true, + observation: { ordinal: 4, pointer: "" }, + response: { ordinal: 3, shape: "identity", pointer: "/created_id" }, + checked_record_ordinals: [5], + }); + }); +}); diff --git a/src/domain/analysisView/environmentStateWrites.ts b/src/domain/analysisView/environmentStateWrites.ts new file mode 100644 index 0000000000..b0c04eebdf --- /dev/null +++ b/src/domain/analysisView/environmentStateWrites.ts @@ -0,0 +1,43 @@ +import type { ExtractedState } from "./environmentStateRules.js"; +import { EnvironmentStateWriteSimulation } from "./EnvironmentStateWriteSimulation.js"; +import type { WriteSimulationInput } from "./environmentStateWriteEffects.js"; + +/** The latest end a record can have and still round up to a whole second a date can hold. */ +const LATEST_END_MS = 8_640_000_000_000_000 - 1000; + +/** The instant a served clock starts from: the end of the last timed record a date can hold, to the next whole second. */ +const clockStart = ( + records: WriteSimulationInput["records"], + timed: boolean, +): string | null => { + if (!timed) return null; + // A record timed past the range of a date carries no usable time, so the clock starts from the others. + const end = records.reduce( + (latest, record) => + Math.abs(record.end) <= LATEST_END_MS + ? Math.max(latest, record.end) + : latest, + Number.NEGATIVE_INFINITY, + ); + return Number.isFinite(end) + ? new Date(Math.ceil(end / 1000) * 1000).toISOString() + : null; +}; + +/** The write plans a capture's own timeline confirms against an extracted state, and the served clock's start. */ +export const extractWrites = ( + state: Pick, + input: Omit< + WriteSimulationInput, + "views" | "types" | "entities" | "documents" + >, +): Pick => ({ + writes: new EnvironmentStateWriteSimulation({ + ...input, + views: new Map(state.views.map((view) => [view.tool, view])), + types: state.types, + entities: state.entities, + documents: state.documents, + }).run(), + write_clock: clockStart(input.records, input.timed), +}); diff --git a/src/server/environment/environmentCapacity.test.ts b/src/server/environment/environmentCapacity.test.ts index 406c572887..ab1300b11d 100644 --- a/src/server/environment/environmentCapacity.test.ts +++ b/src/server/environment/environmentCapacity.test.ts @@ -73,11 +73,9 @@ it("charges environment identity, aggregate tools and escaped text before produc it("reserves the state card only when the state data plane is served", () => { const source = environmentServeSource(); - const manifest = { ...source.manifest, tools: [] }; + const empty = { ...source, manifest: { ...source.manifest, tools: [] } }; const admits = (length: number, dataPlane: "none" | "state") => - environmentToolDefinitions({ ...source, manifest }, "x".repeat(length), { - dataPlane, - }).ok; + environmentToolDefinitions(empty, "x".repeat(length), { dataPlane }).ok; // With no tools, the name and the card are the whole reservation: six bytes per name unit plus quotes. const longest = (dataPlane: "none" | "state") => Math.floor( diff --git a/src/server/environment/serveEnvironment.ts b/src/server/environment/serveEnvironment.ts index fb2ed9fd06..d98958eee2 100644 --- a/src/server/environment/serveEnvironment.ts +++ b/src/server/environment/serveEnvironment.ts @@ -17,7 +17,7 @@ import { import { environmentToolAnswer, type EnvironmentToolAnswer, -} from "../../application/environment/environmentStateAnswer.js"; +} from "../../application/environment/environmentToolAnswer.js"; import { EnvironmentStateStore } from "../../application/environment/environmentStateStore.js"; import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; import { @@ -450,7 +450,7 @@ export const serveEnvironment = async ( options.state === undefined ? undefined : new EnvironmentStateStore(options.state); - const catalog = servedCatalog(source, store !== undefined, name); + const catalog = servedCatalog(source, options.state, name); if (!catalog.ok) return catalog; const { tools, dataPlane } = catalog.value; const card = environmentCard(name, tools, dataPlane); diff --git a/tests/boundary/mcp/environmentServeWritesMcp.test.ts b/tests/boundary/mcp/environmentServeWritesMcp.test.ts new file mode 100644 index 0000000000..046b0cc85c --- /dev/null +++ b/tests/boundary/mcp/environmentServeWritesMcp.test.ts @@ -0,0 +1,97 @@ +import { + Client, + StreamableHTTPClientTransport, +} from "@modelcontextprotocol/client"; +import { describe, expect } from "vitest"; + +import { timedCaptureEvidence } from "../../fixtures/analysisView.js"; +import { + environmentStateServeFixture, + writeEnvironmentServeFiles, +} from "../../fixtures/environmentServe.js"; +import { + BOOK_NOT_FOUND, + BOOK_ROWS, + book, + bookClock, +} from "../../fixtures/environmentStateWrites.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { startListeningProcess } from "../../support/mcp/streamableHttpProcess.js"; +import { processTest } from "../../support/process/processFixture.js"; + +const HTTP_BOUNDARY_TIMEOUT_MS = 90_000; + +/** The book the first served create of "Boundary Book" makes: one second past the served clock's start. */ +const BOUNDARY_BOOK = book(3, "Boundary Book", bookClock(13)); + +describe("rea environment-serve write tools", () => { + processTest( + "applies a served create to the state the next read answers from, until data/reset", + async ({ processes }) => { + const root = await createTestTempDirectory("rea-environment-writes-mcp-"); + const files = await writeEnvironmentServeFiles( + root, + environmentStateServeFixture(timedCaptureEvidence(BOOK_ROWS)), + ); + const serve = [files.manifestPath, "--evidence", files.capturePath]; + const started = await startListeningProcess(processes, [ + "environment-serve", + ...serve, + "--port", + "0", + ]); + const client = new Client({ name: "environment-writes", version: "1" }); + await client.connect(new StreamableHTTPClientTransport(started.url)); + try { + const { tools } = await client.listTools(); + const names = tools.map(({ name }) => name); + expect(names).toEqual( + expect.arrayContaining(["post_books", "patch_books_by_id"]), + ); + // The capture's export write targets no state type, so only confirmed writes are listed. + expect(names).not.toContain("post_exports"); + expect( + tools.find(({ name }) => name === "post_books")?.annotations, + ).toMatchObject({ + readOnlyHint: false, + }); + const created = await client.callTool({ + name: "post_books", + arguments: { body: { name: "Boundary Book" } }, + }); + expect(created.isError).toBeFalsy(); + expect(created.structuredContent).toMatchObject({ + status: 201, + body: BOUNDARY_BOOK, + state: { source: "state", view: "create", id: "3" }, + }); + const read = () => + client.callTool({ name: "get_books_by_id", arguments: { id: "3" } }); + expect((await read()).structuredContent).toMatchObject({ + status: 200, + body: BOUNDARY_BOOK, + state: { source: "state", view: "item", id: "3" }, + }); + const reset = await fetch(new URL("/agentenv", started.url.origin), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + jsonrpc: "2.0", + id: "reset", + method: "data/reset", + }), + }); + await expect(reset.json()).resolves.toMatchObject({ result: {} }); + // The initial state holds no third book, so the read answers the recorded absence. + expect((await read()).structuredContent).toMatchObject({ + status: 404, + body: BOOK_NOT_FOUND, + state: { source: "state", view: "missing", id: "3" }, + }); + } finally { + await client.close(); + } + }, + HTTP_BOUNDARY_TIMEOUT_MS, + ); +}); diff --git a/tests/fixtures/analysisView.ts b/tests/fixtures/analysisView.ts index 30ac1fe239..37f3a7d973 100644 --- a/tests/fixtures/analysisView.ts +++ b/tests/fixtures/analysisView.ts @@ -314,12 +314,14 @@ export const environmentManifestCaptureFixture = ( export const environmentStateCaptureFixture = () => environmentManifestCaptureFixture(environmentStateHar().log.entries); -/** One synthetic exchange: method, path, JSON response body (a string is recorded as written) and status. */ +/** One synthetic exchange: method, path, JSON response body (a string is recorded as written), status, JSON request body and duration in milliseconds. */ export type TimedCaptureRow = readonly [ method: string, path: string, body?: JsonValue, status?: number, + request?: JsonValue, + time?: number, ]; /** Capture Evidence of HAR entries one second apart on one origin, as the environment-state view orders them. */ @@ -328,19 +330,32 @@ export const timedCaptureEvidence = ( ): Evidence => analysisViewCaptureEvidence( environmentManifestCaptureFixture( - rows.map(([method, path, body = null, status = 200], index) => ({ - startedDateTime: new Date( - Date.parse("2026-10-17T18:00:00.000Z") + index * 1000, - ).toISOString(), - time: 5, - request: { method, url: `http://app.example${path}` }, - response: { - status, - content: { - mimeType: "application/json", - text: typeof body === "string" ? body : JSON.stringify(body), + rows.map( + ([method, path, body = null, status = 200, sent, time = 5], index) => ({ + startedDateTime: new Date( + Date.parse("2026-10-17T18:00:00.000Z") + index * 1000, + ).toISOString(), + time, + request: { + method, + url: `http://app.example${path}`, + ...(sent === undefined + ? {} + : { + postData: { + mimeType: "application/json", + text: JSON.stringify(sent), + }, + }), }, - }, - })), + response: { + status, + content: { + mimeType: "application/json", + text: typeof body === "string" ? body : JSON.stringify(body), + }, + }, + }), + ), ), ); diff --git a/tests/fixtures/environmentStateServing.ts b/tests/fixtures/environmentStateServing.ts new file mode 100644 index 0000000000..9b8e687a19 --- /dev/null +++ b/tests/fixtures/environmentStateServing.ts @@ -0,0 +1,46 @@ +import type { ReplayArguments } from "../../src/application/environment/environmentReplay.js"; +import { EnvironmentStateStore } from "../../src/application/environment/environmentStateStore.js"; +import { environmentToolAnswer } from "../../src/application/environment/environmentToolAnswer.js"; +import { projectAnalysisError } from "../../src/domain/analysisErrorProjection.js"; +import { timedCaptureEvidence, type TimedCaptureRow } from "./analysisView.js"; +import { + environmentStateServeFixture, + environmentStateServeInputs, +} from "./environmentServe.js"; +import { BOOK_ROWS } from "./environmentStateWrites.js"; + +/** A served environment over the state derived from a capture of `rows`, answering tool calls as the server does. */ +export const servedFrom = (rows: readonly TimedCaptureRow[] = BOOK_ROWS) => { + const { source, state } = environmentStateServeInputs( + environmentStateServeFixture(timedCaptureEvidence(rows)), + ); + const store = new EnvironmentStateStore(state); + const respond = environmentToolAnswer(source, store); + const call = (name: string, args: ReplayArguments) => { + const tool = source.manifest.tools.find( + (candidate) => candidate.name === name, + ); + if (tool === undefined) throw new Error(`The manifest lacks ${name}`); + return respond(tool, args); + }; + return { + source, + state, + store, + call, + answer: (name: string, args: ReplayArguments = {}) => { + const result = call(name, args); + if (!result.ok) throw result.error; + return result.value; + }, + refusal: (name: string, args: ReplayArguments = {}) => { + const result = call(name, args); + if (result.ok) throw new Error(`Expected ${name} to be refused`); + return projectAnalysisError(result.error); + }, + /** What data/get reports, as text, so whole states compare byte for byte. */ + snapshot: () => JSON.stringify(store.snapshot()), + }; +}; + +export type Served = ReturnType; diff --git a/tests/fixtures/environmentStateWrites.ts b/tests/fixtures/environmentStateWrites.ts new file mode 100644 index 0000000000..0880edbd0d --- /dev/null +++ b/tests/fixtures/environmentStateWrites.ts @@ -0,0 +1,165 @@ +import { + projectAnalysisView, + type AnalysisViewResult, +} from "../../src/domain/analysisView/analysisView.js"; +import type { TemplateRule } from "../../src/domain/analysisView/environmentManifestRules.js"; +import type { Evidence } from "../../src/domain/evidence.js"; +import type { JsonValue } from "../../src/domain/jsonValue.js"; +import { + analysisViewCaptureEvidence, + environmentManifestCaptureFixture, + type TimedCaptureRow, +} from "./analysisView.js"; + +/** When the first synthetic exchange starts; exchange `i` starts `i` seconds later, as in `timedCaptureEvidence`. */ +const CAPTURE_START_MS = Date.parse("2026-10-17T18:00:00.000Z"); + +/** The instant `seconds` after the first exchange, spelled as the books application spells clock values. */ +export const bookClock = (seconds: number): string => + new Date(CAPTURE_START_MS + seconds * 1000) + .toISOString() + .replace(".000Z", "Z"); + +/** Lower case with every run of other characters joined by one `-`, as the books application slugs names. */ +const slugOf = (name: string): string => + name + .toLowerCase() + .replace(/[^a-z0-9]+/gu, "-") + .replace(/^-|-$/gu, ""); + +/** A book as the application shows it. */ +export type Book = { [field: string]: JsonValue }; + +/** A book whose slug follows its name. */ +export const book = ( + id: number, + name: string, + created: string, + updated = created, +): Book => ({ + id, + name, + slug: slugOf(name), + created_at: created, + updated_at: updated, +}); + +/** A book renamed by an update recorded at exchange `seconds`: its slug and update clock follow. */ +export const renamedBook = ( + entity: Book, + name: string, + seconds: number, +): Book => ({ + ...entity, + name, + slug: slugOf(name), + updated_at: bookClock(seconds), +}); + +export const FIRST_BOOK = book(1, "First Book", "2026-10-01T09:00:00Z"); +export const SECOND_BOOK = book(2, "Second Book", "2026-10-01T10:00:00Z"); +/** The book the recorded create at exchange 3 made. */ +export const THIRD_BOOK = book(3, "Third Book", bookClock(3)); +/** The first book after the recorded update at exchange 5. */ +export const RENAMED_BOOK = renamedBook(FIRST_BOOK, "Renamed Book", 5); + +/** What the item read answered for a book the application does not hold. */ +export const BOOK_NOT_FOUND = { detail: "Not found." }; +/** What the delete answered for a book the application does not hold. */ +export const DELETE_NOT_FOUND = { detail: "No Book matches the given query." }; + +/** Reads before any write: the whole list, then each book; the initial state comes from these. */ +export const BOOK_INIT_ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], +]; + +/** + * A create, an update and a delete, each confirmed by later reads; a delete of a book the + * application did not hold; and a write to no state type. + */ +export const BOOK_ROWS: readonly TimedCaptureRow[] = [ + ...BOOK_INIT_ROWS, + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + ["GET", "/api/books/3", THIRD_BOOK], + ["PATCH", "/api/books/1", RENAMED_BOOK, 200, { name: "Renamed Book" }], + ["GET", "/api/books/1", RENAMED_BOOK], + ["DELETE", "/api/books/3", "", 204], + ["GET", "/api/books/3", BOOK_NOT_FOUND, 404], + ["GET", "/api/books", [RENAMED_BOOK, SECOND_BOOK]], + ["DELETE", "/api/books/9", DELETE_NOT_FOUND, 404], + ["POST", "/api/exports", { queued: true }, 202, { format: "csv" }], +]; + +/** The end of the last exchange of BOOK_ROWS, to the next whole second: where the served clock starts. */ +export const BOOK_WRITE_CLOCK = "2026-10-17T18:00:12.000Z"; + +/** + * Capture Evidence like `timedCaptureEvidence`, but each request body is recorded under the + * given media type (a string as written, anything else as JSON), and the exchanges may carry + * no start times. + */ +export const rawCaptureEvidence = ( + rows: readonly TimedCaptureRow[], + request: { readonly mimeType: string; readonly timed: boolean }, +): Evidence => + analysisViewCaptureEvidence( + environmentManifestCaptureFixture( + rows.map(([method, path, body = null, status = 200, sent], index) => ({ + ...(request.timed + ? { + startedDateTime: new Date( + CAPTURE_START_MS + index * 1000, + ).toISOString(), + time: 5, + } + : {}), + request: { + method, + url: `http://app.example${path}`, + ...(sent === undefined + ? {} + : { + postData: { + mimeType: request.mimeType, + text: typeof sent === "string" ? sent : JSON.stringify(sent), + }, + }), + }, + response: { + status, + content: { + mimeType: "application/json", + text: typeof body === "string" ? body : JSON.stringify(body), + }, + }, + })), + ), + ); + +type EnvironmentStateView = Extract< + AnalysisViewResult, + { kind: "environment-state" } +>; + +/** The environment-state view a capture projects with no opt-in template rules. */ +export const environmentStateViewOf = ( + capture: Evidence, + templateRules: readonly TemplateRule[] = [], +): EnvironmentStateView => { + const result = projectAnalysisView( + { + evidenceId: capture.evidence_id, + operation: capture.operation, + normalizedResult: capture.normalized_result, + subject: capture.subject, + limitations: [], + parameters: capture.parameters, + }, + { kind: "environment-state", template_rules: [...templateRules] }, + ); + if (!result.ok || result.value.kind !== "environment-state") + throw new Error("Expected an environment state"); + return result.value; +};