From a0150bde68e1b5593920dcca1216c785b72ca22f Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Fri, 28 Aug 2026 23:46:37 -0700 Subject: [PATCH 1/2] perf(redis): skip refills behind observed watermarks --- README.md | 44 ++++---- scripts/test-package.mjs | 72 ++++++++++-- src/dialcache.ts | 48 ++++++-- src/index.ts | 2 + src/internal/cache-result.ts | 3 +- src/internal/redis-cache.ts | 124 ++++++++++++++++----- src/internal/redis-payload.ts | 61 ++++++++-- src/metrics.ts | 1 + src/node-redis.ts | 7 +- src/redis-client.ts | 55 +++++++-- src/redis-protocol.ts | 7 +- src/valkey-glide.ts | 7 +- test/datadog.test.ts | 1 + test/dialcache-compression.test.ts | 50 ++++++++- test/dialcache-invalidation.test.ts | 91 +++++++++++++-- test/dialcache-redis.test.ts | 39 +++++++ test/dialcache-shadow-confirmation.test.ts | 100 +++++++++++++++-- test/dialcache-stale-on-error.test.ts | 8 +- test/fake-redis.ts | 37 +++--- test/node-redis.test.ts | 31 ++++++ test/prometheus.test.ts | 1 + test/redis-cluster.integration.test.ts | 15 ++- test/redis-payload.test.ts | 43 +++++++ test/redis-real.integration.test.ts | 120 ++++++++++++++++---- test/valkey-glide.test.ts | 31 ++++++ 25 files changed, 839 insertions(+), 159 deletions(-) diff --git a/README.md b/README.md index ef5d955..6874983 100644 --- a/README.md +++ b/README.md @@ -408,19 +408,19 @@ Awaiting those public promises does not drain detached shadow work. Shadow sched Neither adapter owns additional resources or native script handles, so the application simply closes the underlying client after draining work. -Reads use native `GET` for untracked entries and one atomic `MGET` for each tracked value-and-watermark pair. The adapters validate and decode the returned frame in the Node process. Tracked reads are deliberately routed to primaries so a lagging replica cannot hide an invalidation watermark. After a read settles, DialCache evaluates the frame against the observing application's `Date.now()`. Without stale-on-error, the read accepts only nonnegative ages strictly below the effective remote TTL `F`. With a positive recovery maximum `M`, that same initial read is bounded by `M`: ages below `F` deserialize and serve normally, while ages from `F` through strictly below `M` remain raw as a possible source-error recovery candidate. Future-dated frames fail closed before deserialization and emit the bounded offset observation described under [Metrics](#metrics). A shadow confirmation read still observes a future offset but retains the payload only for supersession comparison; it can never serve that frame. +Reads use native `GET` for untracked entries and one atomic `MGET` for each tracked value-and-watermark pair. The adapters validate and decode the returned frame in the Node process. Tracked reads are deliberately routed to primaries so a lagging replica cannot hide an invalidation watermark. When a tracked semantic miss observes a present, valid numeric watermark, the bundled adapters return a typed `RedisWatermarkMiss` carrying that watermark; generic and legacy misses remain `null`. After a read settles, DialCache evaluates the frame against the observing application's `Date.now()`. Without stale-on-error, the read accepts only nonnegative ages strictly below the effective remote TTL `F`. With a positive recovery maximum `M`, that same initial read is bounded by `M`: ages below `F` deserialize and serve normally, while ages from `F` through strictly below `M` remain raw as a possible source-error recovery candidate. Future-dated frames fail closed before deserialization and emit the bounded offset observation described under [Metrics](#metrics). A shadow confirmation read still observes a future offset but retains the payload only for supersession comparison; it can never serve that frame. -Writes are native too, so the payload never crosses the Redis-to-Lua boundary. Each bundled adapter samples the writer process's `Date.now()` once immediately before dispatch and issues one `SET valueKey frame PX cacheTtlMs` containing the complete version-1 frame. Tracked and untracked writes have the same adapter request and command shape; only tracked reads include the watermark key. With stale-on-error enabled, the requested physical TTL is `M` instead of `F`. `M` remains the configured logical recovery ceiling even when it exceeds one hour. Core separately caps every tracked Redis value's physical TTL at one hour, so such a tracked candidate may be evicted by expiry before it reaches logical age `M`; untracked Redis and local TTLs retain their configured limits. Each dispatched tracked write whose requested TTL exceeds that cap emits `error="tracked_ttl_clamped"`. The write never reads, creates, or extends a watermark. A frame written behind an active watermark remains physically present but is a tracked miss until its `createdAtMs` is greater than the watermark. Same-key writes are ordinary Redis last-writer-wins operations, with no Lua, pipeline, or transaction on the write path. +Writes are native too, so the payload never crosses the Redis-to-Lua boundary. For a DialCache fill following a typed tracked miss, core samples one candidate `createdAtMs = Date.now()` after the fallback succeeds and before `serializer.dump`, compression, or frame construction. If that candidate is at or before the observed watermark, DialCache skips the known-fenced refill at that point: it does not invoke the serializer, compress or allocate the payload/frame, or dispatch `SET`. Otherwise core passes the same candidate through optional `RedisWriteRequest.createdAtMs`, and the bundled adapter encodes that exact value. Ordinary, untracked, missing/malformed-watermark, and legacy-adapter misses leave the optional timestamp absent, preserving the adapter-side `Date.now()` sample immediately before dispatch. Every dispatched write issues one `SET valueKey frame PX cacheTtlMs` containing the complete version-1 frame; only tracked reads include the watermark key. With stale-on-error enabled, the requested physical TTL is `M` instead of `F`. `M` remains the configured logical recovery ceiling even when it exceeds one hour. Core separately caps every tracked Redis value's physical TTL at one hour, so such a tracked candidate may be evicted by expiry before it reaches logical age `M`; untracked Redis and local TTLs retain their configured limits. Each dispatched tracked write whose requested TTL exceeds that cap emits `error="tracked_ttl_clamped"`. The write never reads, creates, or extends a watermark. A dispatched frame can still be fenced if the watermark advances after the read snapshot. Same-key writes are ordinary Redis last-writer-wins operations, with no Lua, pipeline, or transaction on the write path. -The network shape remains one top-level Redis command and one round trip per semantic read (`GET` or `MGET`) and one `SET` per write. Stale recovery reuses the frame returned by that initial command and never adds a second Redis read, including after a source rejection. Retaining a raw candidate instead consumes process memory until that source attempt settles, once per distinct in-flight key (same-key coalesced callers share it). DialCache does not call Redis `TIME` or maintain a Redis-clock offset. Use the maintainer benchmarks below to measure the target Redis/Valkey version, payload distribution, and high-cardinality in-flight memory exposure. +The network shape remains one top-level Redis command and one round trip per semantic read (`GET` or `MGET`) and one `SET` per dispatched write. Stale recovery reuses the frame returned by that initial command and never adds a second Redis read, including after a source rejection. Retaining a raw candidate instead consumes process memory until that source attempt settles, once per distinct in-flight key (same-key coalesced callers share it). Conditional refill suppression reuses the existing tracked `MGET` result and adds no command or round trip. DialCache does not call Redis `TIME` or maintain a Redis-clock offset. Use the maintainer benchmarks below to measure the target Redis/Valkey version, payload distribution, and high-cardinality in-flight memory exposure. -Native commands retain Redis's wrong-type behavior. An untracked `GET` surfaces `WRONGTYPE`; tracked `MGET` represents a wrong-type member as a missing value. A wrong-type tracked value is therefore a semantic miss and may be replaced with a valid complete frame after fallback. A wrong-type watermark is indistinguishable from an absent watermark to `MGET`, so tracked reads apply the zero baseline; it does not prevent or alter native value writes. The next explicit invalidation replaces that wrong-type key with a valid string watermark. Other script read failures remain errors and cannot bypass the monotonic update. +Native commands retain Redis's wrong-type behavior. An untracked `GET` surfaces `WRONGTYPE`; tracked `MGET` represents a wrong-type member as a missing value. A wrong-type tracked value is therefore a semantic miss; when the paired watermark is a valid numeric string, the miss carries that fence and may skip a known-dead refill, while an absent or wrong-type watermark yields the ordinary zero-baseline behavior. A wrong-type watermark is indistinguishable from an absent watermark to `MGET`; it does not prevent or alter native value writes. The next explicit invalidation replaces that wrong-type key with a valid string watermark. Other script read failures remain errors and cannot bypass the monotonic update. Node-redis forces tracked cluster commands to the slot primary. GLIDE uses an explicit primary route in cluster mode; in standalone mode it sends `MGET` through a one-command non-atomic batch because direct read commands follow the client's replica-read preference. Standalone batches use the primary, and `MGET` itself provides the atomic snapshot without consuming caller-owned `WATCH` state. The GLIDE helper distinguishes those modes from the direct client's runtime identity and rejects ambiguous clients instead of silently choosing a route. Invalidation is the only remaining Lua operation. Both adapters dispatch it as `EVALSHA` by the script source's SHA1 and retry a rejected dispatch once by re-sending the source as `EVAL`. The script is idempotent: its watermark advances monotonically and its TTL only widens, so a duplicate execution after an ambiguous failure is harmless. Reply-domain violations are deterministic and are not retried. If the retry also fails, GLIDE attaches the original rejection as `cause` when possible; node-redis surfaces the retry rejection unmodified because disconnect failures may be shared across callers. A healed retry is indistinguishable from first-attempt success in DialCache metrics. Monitor server-side `INFO commandstats` for unexpected `EVAL` volume or rejected `EVALSHA` calls. -Command-restricted Redis ACLs must allow native `GET`, `MGET`, and `SET`, plus `EVALSHA` and `EVAL` for invalidation recovery. If script-invoked commands are checked separately, the invalidation script needs `GET`, `SET`, and `PTTL`. Redis `TIME`, `MULTI`, `EXEC`, `WATCH`, `UNLINK`, and `SCRIPT LOAD` are not used by DialCache. The integration matrix covers Redis 6.2 and Valkey 8. +Command-restricted Redis ACLs must allow native `GET`, `MGET`, and `SET`, plus `EVALSHA` and `EVAL` for invalidation recovery. If script-invoked commands are checked separately, the invalidation script needs `GET`, `SET`, and `PTTL`. Redis `TIME`, `MULTI`, `EXEC`, `WATCH`, `UNLINK`, and `SCRIPT LOAD` are not used by DialCache. Conditional refill suppression reuses the existing tracked `MGET` result and adds no command or round trip. The integration matrix covers Redis 6.2 and Valkey 8. #### Stale on source error @@ -482,9 +482,9 @@ Writes, invalidations, async `cacheConfigProvider` calls, and custom serializer #### Serialization -The core Redis boundary is the client-agnostic `DialCacheRedisClient` interface. Writes accept serialized values as `string | Buffer`; reads return a `DecodedRedisFrame` — the decoded `string | Buffer` payload plus the frame header's writer-client `createdAtMs` — and the interface does not expose client commands or wire encodings. That timestamp is correctness-relevant: custom clients must return the frame's valid nonnegative safe-integer epoch timestamp rather than a constant. `RedisWriteRequest` contains only `valueKey`, `cacheTtlMs`, and `value`, and `write()` returns `void`; watermark ownership is exclusive to tracked reads and invalidation. +The core Redis boundary is the client-agnostic `DialCacheRedisClient` interface. Writes accept serialized values as `string | Buffer`; reads return `RedisReadResult`, which is a `DecodedRedisFrame` — the decoded `string | Buffer` payload plus the frame header's writer-client `createdAtMs` — a typed `RedisWatermarkMiss`, or `null`. The interface does not expose client commands or wire encodings. A decoded frame's timestamp is correctness-relevant: custom clients must return the frame's valid nonnegative safe-integer epoch timestamp rather than a constant. `RedisWriteRequest` contains `valueKey`, `cacheTtlMs`, `value`, and optional `createdAtMs`, and `write()` returns `void`; watermark ownership remains exclusive to tracked reads and invalidation. Core supplies `createdAtMs` only when a typed miss makes that exact candidate part of the refill decision. A custom client that continues returning only `DecodedRedisFrame | null` remains correct and source-compatible, but does not enable the known-fenced refill optimization. A custom client that returns `RedisWatermarkMiss` opts into that optimization and must encode a supplied `RedisWriteRequest.createdAtMs` exactly so the decision timestamp and stored frame cannot diverge. -The shared `encodeRedisFrame`, `decodeRedisFrame`, and `decodeTrackedRedisFrame` helpers, the `validateRedisSetReply` and `validateRedisScriptInvalidationReply` reply helpers, the `ceilSupportedCacheTtlMs` TTL guard, and the invalidation Lua source are available from `dialcache/redis-protocol`. A custom write samples `Date.now()` once, calls `encodeRedisFrame(value, createdAtMs)`, and sends one `SET valueKey frame PX cacheTtlMs` after validating the TTL. A custom tracked read atomically obtains `[value, watermark]` from the primary and passes both replies to `decodeTrackedRedisFrame`; a missing watermark is treated as zero. Invalidation passes `KEYS = [watermarkKey]` and `ARGV = [futureBufferMs, invalidatedAtMs]`, with one stable client timestamp reused across retries. Custom adapters can throw the root-exported `DialCacheRedisPayloadError`, `DialCacheRedisPayloadEncodingError`, and `DialCacheRedisProtocolError` classes to distinguish malformed replies, unsupported encodings, and reply-domain violations. DialCache records bounded `cache_read`, `cache_write`, or `invalidation` metrics by failure site. +The shared `encodeRedisFrame`, `decodeRedisFrame`, `decodeTrackedRedisFrame`, and `decodeTrackedRedisReadResult` helpers, the `validateRedisSetReply` and `validateRedisScriptInvalidationReply` reply helpers, the `ceilSupportedCacheTtlMs` TTL guard, and the invalidation Lua source are available from `dialcache/redis-protocol`. A custom write chooses `const createdAtMs = request.createdAtMs === undefined ? Date.now() : request.createdAtMs`, calls `encodeRedisFrame(request.value, createdAtMs)`, and sends one `SET valueKey frame PX cacheTtlMs` after validating the TTL. The fallback clock covers ordinary core writes and direct callers that omit the optional field; supplied values must not be resampled or replaced, and invalid runtime values must still be rejected by the frame encoder. A custom tracked read atomically obtains `[value, watermark]` from the primary. Passing both replies to `decodeTrackedRedisReadResult` opts into typed watermark misses; the backward-compatible `decodeTrackedRedisFrame` collapses those misses to `null` and preserves the established `DecodedRedisFrame | null` surface. A missing watermark is treated as zero for valid frames, while a malformed numeric watermark fails closed as a generic miss. Invalidation passes `KEYS = [watermarkKey]` and `ARGV = [futureBufferMs, invalidatedAtMs]`, with one stable client timestamp reused across retries. Custom adapters can throw the root-exported `DialCacheRedisPayloadError`, `DialCacheRedisPayloadEncodingError`, and `DialCacheRedisProtocolError` classes to distinguish malformed replies, unsupported encodings, and reply-domain violations. DialCache records bounded `cache_read`, `cache_write`, or `invalidation` metrics by failure site. Redis values use a compact binary frame: @@ -495,7 +495,7 @@ byte 10 payload encoding (0 = UTF-8, 1 = raw binary) bytes 11... serialized payload (optionally zstd-compressed; see Compression) ``` -Adapters build complete frames in the Node process with `encodeRedisFrame` and decode them with Node buffer primitives. The version-1 value-envelope format, Redis value-key derivation, and decimal watermark encoding are unchanged. That wire compatibility does not make old tracked state safe to carry across the protocol cutover: old watermark lifetimes were derived for the old write protocol. +Adapters build complete frames in the Node process with `encodeRedisFrame` and decode them with Node buffer primitives. `RedisWatermarkMiss` exists only across the in-process semantic adapter boundary; it is not a new Redis value. The version-1 value-envelope format, Redis value-key derivation, decimal watermark encoding, tracked `MGET`, and dispatched `SET` are unchanged. That wire compatibility does not make old tracked state safe to carry across the protocol cutover: old watermark lifetimes were derived for the old write protocol. Redis physical TTL bounds how long the stored key remains available to future reads. A completed read owns its returned frame, so later expiry, deletion, or eviction cannot revoke that in-process snapshot. The frame timestamp enforces logical `F`/`M` age, future-frame rejection, and shadow value-age observability. `payload` is produced by the operation's serializer, or by `JsonSerializer` by default. Custom serializers can return either `string` or `Buffer`. Payloads stored raw keep their exact serialized bytes: strings are stored as UTF-8 and Buffers byte-for-byte without base64 expansion, except that binary output beginning with a [compression envelope byte](#compression) (`0x00`–`0x02`) gains a one-byte escape prefix on the wire. Payloads at or above the compression threshold may instead be stored as a zstd envelope (see [Compression](#compression)), so wire bytes for large values are not the serializer's output. Adapters return the frame payload as-is; the envelope — including restoring a compressed string's representation before `serializer.load` — is interpreted by the core above them. @@ -539,7 +539,7 @@ Alternatively, keep all traffic disabled until every old tracked value and water #### Compression -DialCache transparently compresses serialized Redis payloads with zstd (level 3, via `node:zlib`) when they are at least 4096 serialized bytes, and stores the compressed form only when it is smaller than the raw stored form. Compression sits below the serializer and above the Redis client, so serializers, adapters, and the frame layout are unaffected. The first byte of a binary frame payload written by a release with payload compression is an envelope byte: `0x01` marks a compressed UTF-8 string and `0x02` compressed binary output, each followed by the zstd frame, while `0x00` is an escape prefix for raw binary serializer output whose own first byte is `0x00`–`0x02` (readers strip the prefix and never decompress it; the escape applies even with `compression: false`). Payloads below the threshold are otherwise stored byte-identical to earlier DialCache releases; only binary output beginning with an envelope byte gains the one-byte escape. +DialCache transparently compresses serialized Redis payloads with zstd (level 3, via `node:zlib`) when they are at least 4096 serialized bytes, and stores the compressed form only when it is smaller than the raw stored form. Compression sits below the serializer and above the Redis client, so serializers, adapters, and the frame layout are unaffected. A refill already known to be fenced is rejected before `serializer.dump` reaches this layer, avoiding compression and large intermediate payload/frame allocation as well as the native `SET`. The first byte of a binary frame payload written by a release with payload compression is an envelope byte: `0x01` marks a compressed UTF-8 string and `0x02` compressed binary output, each followed by the zstd frame, while `0x00` is an escape prefix for raw binary serializer output whose own first byte is `0x00`–`0x02` (readers strip the prefix and never decompress it; the escape applies even with `compression: false`). Payloads below the threshold are otherwise stored byte-identical to earlier DialCache releases; only binary output beginning with an envelope byte gains the one-byte escape. Decompressed payloads are capped at 512 MiB, mirrored on the write side by refusing to compress anything larger, so no writable entry is unreadable and a corrupt or hostile entry cannot force a giant synchronous allocation. zstd runs synchronously on the event loop: at level 3 it stays cheaper than the adjacent `JSON.stringify`/`parse` at every size (~2 ms to compress 2 MiB), but cost rises steeply with level — measured ~250 ms for 1 MiB at level 19 and ~1.5 s for 2 MiB at level 22 — so treat high levels as an informed opt-in and watch the compression timer metric. @@ -615,16 +615,16 @@ On a served hit, DialCache returns the already-decoded cached value before start The detached job uses this bounded algorithm: 1. Obtain the original Redis payload as `C0` using the key's existing tracked or untracked read mode. -2. If `C0` is missing, wait for the caller's successfully accepted `S` after its configured fallback boundary and attempt one normal Redis write using the resolved TTL. Before the whole-job deadline, emit `filled` when Redis accepts it or `fill_error` when serialization or the write fails. A tracked fill can be physically stored yet remain fenced on later tracked reads. +2. If `C0` is missing, wait for the caller's successfully accepted `S` after its configured fallback boundary. For a typed tracked miss, sample the fill timestamp before serialization: emit `fill_fenced` and stop when that candidate is at or before the observed watermark, without invoking `serializer.dump`, compression, or Redis; otherwise attempt one normal Redis write using the resolved TTL and that exact candidate. For a generic `null` miss, attempt the ordinary write without supplying a core timestamp, preserving adapter-side sampling. Before the whole-job deadline, emit `filled` when Redis accepts the write or `fill_error` when serialization or the write fails. A tracked fill can still be physically stored yet remain fenced if the watermark advances after `C0`. 3. If `C0` is non-null, obtain `S`, deserialize an isolated snapshot of `C0`, and run the default or custom semantic comparator. Any non-null `C0` is observation-only: DialCache never repairs or overwrites it, including when deserialization fails. 4. If `C0` and `S` match semantically, emit `match` without another Redis read. 5. Otherwise, reread Redis directly in the same mode as `C1`, bypassing request-local and process-local cache. 6. If `C1` is missing under the normal value/watermark protocol or differs byte-for-byte from `C0`, emit `superseded`; if it is identical, emit `mismatch`. A future-dated `C1` records its offset but remains available for this payload comparison, so a reader-clock step does not change the verdict. 7. If the confirmation read fails or reaches its Redis-read deadline, emit `confirmation_error`. -Here a semantic miss means the Redis read returned `null`; it does not include a non-null payload that later fails deserialization. A physical tracked frame rejected by its watermark, timestamp domain, or future-time check is therefore a miss and may be replaced by the fill. A caller fallback rejection or timeout never becomes accepted `S` and never starts the fill. +Here a semantic miss means the Redis read returned `null` or a typed `RedisWatermarkMiss`; it does not include a non-null payload that later fails deserialization. A physical tracked frame rejected by its watermark, timestamp domain, or future-time check is therefore a miss. Only a typed miss carries a trustworthy observed fence and can produce `fill_fenced`; generic or legacy `null` misses retain the normal refill behavior. A caller fallback rejection or timeout never becomes accepted `S` and never starts the fill. -Both detached Redis reads use the effective `remoteReadTimeoutMs` and the key's normal native `GET` or `MGET` protocol. The initial `C0` observation enforces the logical remote TTL `F`; the non-serving `C1` confirmation bypasses logical age solely to determine whether the original payload bytes were superseded. Every semantic-miss fill uses the same serializer, complete-frame SET, and client-clock timestamp semantics as an ordinary fill; when stale-on-error is active, it requests physical retention through `M` while serving reads still enforce `F`. Tracked `C0` and `C1` reads remain watermark-aware and are routed to primaries by the bundled adapters. The fill itself never reads or mutates the watermark. Untracked reads use the ordinary one-key read route, which has no shadow-specific primary guarantee. Strings compare exactly, Buffers compare by bytes, and string/Buffer pairs compare by their UTF-8 bytes. DialCache does not deserialize `C1`, compare it with `S`, or chase another version. +Both detached Redis reads use the effective `remoteReadTimeoutMs` and the key's normal native `GET` or `MGET` protocol. The initial `C0` observation enforces the logical remote TTL `F`; the non-serving `C1` confirmation bypasses logical age solely to determine whether the original payload bytes were superseded. Every dispatched semantic-miss fill uses the same serializer, resolved physical TTL, and complete-frame `SET` as the caller path; when stale-on-error is active, it requests physical retention through `M` while serving reads still enforce `F`. A typed watermark miss uses the same exact candidate-timestamp pairing and fence decision as its caller-path counterpart; generic `null` fills preserve the ordinary adapter-side timestamp sample. Tracked `C0` and `C1` reads remain watermark-aware and are routed to primaries by the bundled adapters. The fill itself never issues another watermark read or mutates the watermark; `fill_fenced` is decided from the original `C0` snapshot and adds no round trip. Untracked reads use the ordinary one-key read route, which has no shadow-specific primary guarantee. Strings compare exactly, Buffers compare by bytes, and string/Buffer pairs compare by their UTF-8 bytes. DialCache does not deserialize `C1`, compare it with `S`, or chase another version. The detached scheduler, Redis-read deadline timers, and overall shadow deadline timer are unreferenced, so they do not keep an otherwise idle process alive. Detachment is asynchronous work on the Node event loop, not a worker thread: synchronous source, serializer, or comparator work can still occupy the event loop after the request path has been released. @@ -634,7 +634,7 @@ Detached execution retains the original `cached()` argument references or `getOr Each job has one monotonic deadline across detached `C0`, the SoT result, serializer work, comparison, optional `C1`, and semantic-miss fill. Served-hit timing begins when its detached validation callback starts. On a ramped-down path, timing begins immediately before the caller's SoT invocation so synchronous source work that runs before admission still consumes the same budget. Each Redis read also has its effective read deadline. A finite `fallbackTimeoutMs` is reused as the overall shadow budget. When `fallbackTimeoutMs` is `null`, the normal fallback remains intentionally unbounded, but detached shadow work still uses a 60-second budget. Once timeout delivery marks a job abandoned, DialCache releases retained `C0` references and prevents later phases from starting. -JavaScript promises and Redis writes do not provide a general cancellation or transaction boundary. Work already dispatched may continue and keeps the shadow slot until it settles. A write rejection, `fill_error`, or shadow `timeout` after dispatch does not prove that Redis was unchanged; the command may have executed before its result became unavailable. Conversely, `filled` means the semantic client returned success before the shadow deadline, not that the value is still present. Give dependencies finite native budgets and treat shadow outcomes as best-effort operational evidence. +JavaScript promises and Redis writes do not provide a general cancellation or transaction boundary. Work already dispatched may continue and keeps the shadow slot until it settles. A write rejection, `fill_error`, or shadow `timeout` after dispatch does not prove that Redis was unchanged; the command may have executed before its result became unavailable. Conversely, `filled` means the semantic client returned success before the shadow deadline, not that the value is still present. `fill_fenced` is stronger but narrower evidence: DialCache locally skipped dispatch because its candidate timestamp was already at or before the valid watermark observed by `C0`; it does not prove that Redis stayed unchanged afterward. Give dependencies finite native budgets and treat shadow outcomes as best-effort operational evidence. A `match` means application-level value equality: @@ -646,9 +646,9 @@ DialCache retains the semantic frame returned by the Redis client — its `strin The effective serializer's `load` method therefore runs a second time for a sampled served hit and once in detached work for a shadow-only hit. It must be repeatable, non-mutating, and return independently usable values. On a semantic miss, its `dump` method may run after the caller has received `S`, so `S` must remain immutable through detached serialization. A custom `DialCacheRedisClient` must return an operation-owned frame whose string/Buffer payload contents remain stable after `read()` settles. Comparing the deserialized cached snapshot with the raw source value intentionally detects lossy serialization; use a custom comparator only when such normalization or ignored fields are valid use-case semantics. -Shadow metrics use the bounded outcomes `match`, `mismatch`, `superseded`, `filled`, `fill_error`, `redis_error`, `source_error`, `deserialization_error`, `comparison_error`, `confirmation_error`, `timeout`, and `dropped`. `redis_error` applies to the initial shadow-only `C0` read and `confirmation_error` applies to `C1`. A semantic `C0` miss is an ordinary `miss{layer="remote_shadow"}` and terminates with a fill, source, or timeout outcome rather than a second shadow outcome for the miss itself. Labels never contain cache ids, values, payloads, Redis keys, or raw exception text. +Shadow metrics use the bounded outcomes `match`, `mismatch`, `superseded`, `filled`, `fill_fenced`, `fill_error`, `redis_error`, `source_error`, `deserialization_error`, `comparison_error`, `confirmation_error`, `timeout`, and `dropped`. `redis_error` applies to the initial shadow-only `C0` read and `confirmation_error` applies to `C1`. A semantic `C0` miss is an ordinary `miss{layer="remote_shadow"}` and terminates with `filled`, `fill_fenced`, `fill_error`, `source_error`, or `timeout` rather than a second shadow outcome for the miss itself. Labels never contain cache ids, values, payloads, Redis keys, or raw exception text. -A `match` or `mismatch` verdict additionally records the validated value's age through the optional `observeShadowValueAge` adapter hook: the observing process's epoch clock minus the writer-stamped frame's `createdAtMs` (the served `C0` frame, or the detached `C0` frame on a ramped-down path), in seconds, clamped at zero. The age is captured at verdict time, so a mismatch age lands one confirmation read after the comparison itself. A confirmed `mismatch` age therefore measures how long the stale value had been readable when validation caught it. Because writer and observer are different application processes, their clock offset makes the age coarse operational evidence rather than a precise measurement. Outcomes that deliver no verdict on a retained value — including `superseded`, `filled`, and every error or timeout outcome — record no age. The hook does not gate shadow eligibility; only `shadowValidation` does. +A `match` or `mismatch` verdict additionally records the validated value's age through the optional `observeShadowValueAge` adapter hook: the observing process's epoch clock minus the writer-stamped frame's `createdAtMs` (the served `C0` frame, or the detached `C0` frame on a ramped-down path), in seconds, clamped at zero. The age is captured at verdict time, so a mismatch age lands one confirmation read after the comparison itself. A confirmed `mismatch` age therefore measures how long the stale value had been readable when validation caught it. Because writer and observer are different application processes, their clock offset makes the age coarse operational evidence rather than a precise measurement. Outcomes that deliver no verdict on a retained value — including `superseded`, `filled`, `fill_fenced`, and every error or timeout outcome — record no age. The hook does not gate shadow eligibility; only `shadowValidation` does. Confirmed-mismatch logging is separately opt-in through `shadow.logMismatches`; it never replaces the required `shadowValidation` metric or emits for a mismatch candidate that becomes `superseded`. The single warning contains `cacheNamespace`, `useCase`, `keyType`, `outcome: "mismatch"`, `cacheKey`, `cachedValueJson`, and `sourceValueJson`. `cacheKey` is the logical DialCache URN capped at 2 KiB, not the physical Redis storage key. DialCache independently applies native `JSON.stringify` to the deserialized cached snapshot and raw source value supplied to the comparator, then caps each resulting string at 8 KiB. A byte-clipped field ends in `...[truncated]`, counted inside its cap. If native JSON throws or returns `undefined`, the corresponding JSON field is `null`; the other side is still attempted. DialCache does not compute a textual diff or call the configured serializer again for logging. @@ -658,13 +658,13 @@ The byte caps apply before logger framing or escaping, so they do not guarantee Detached Redis reads, serializer loads/dumps, payload sizes, and read/write errors use the existing layer label with `layer="remote_shadow"`. This distinguishes non-serving Redis cost from caller-path `layer="remote"` telemetry without adding a metric or label. The established `observeGet{layer="remote"}` boundary includes caller-path deserialization, while `observeGet{layer="remote_shadow"}` ends when the deadline-bounded Redis read result settles; detached serializer work and any later raw-client settlement are outside that timer. The request-path read that supplied a served `C0` keeps `layer="remote"`, and a ramped-down caller keeps `disabled{layer="remote", reason="ramped_down"}`. No `disabled{layer="remote_shadow"}` event is emitted for ineligible or dropped work; `dropped` remains the terminal shadow outcome. Confirmation reads use the same `remote_shadow` value, with `superseded` or `confirmation_error` describing their role. -The command amplification is bounded: a selected served hit adds one SoT read and adds `C1` only for a semantic mismatch candidate; a selected ramped-down hit adds detached `C0`, reuses the caller's existing SoT read, and likewise adds `C1` only for a candidate; a selected ramped-down miss adds detached `C0` and at most one fill in the key's existing mode. `superseded` means only that the original observation could not be confirmed. `mismatch` means the exact `C0` payload survived another Redis read after the SoT disagreement; it is not a cross-system atomic snapshot or a guarantee that the mismatch persists. For an untracked key it is also not proof of invalidation safety because no watermark participates. +The command amplification is bounded: a selected served hit adds one SoT read and adds `C1` only for a semantic mismatch candidate; a selected ramped-down hit adds detached `C0`, reuses the caller's existing SoT read, and likewise adds `C1` only for a candidate; a selected ramped-down miss adds detached `C0` and at most one fill in the key's existing mode. A known-fenced tracked fill stops before serialization and adds no `SET`; no path adds a fence-check command. `superseded` means only that the original observation could not be confirmed. `mismatch` means the exact `C0` payload survived another Redis read after the SoT disagreement; it is not a cross-system atomic snapshot or a guarantee that the mismatch persists. For an untracked key it is also not proof of primary freshness or invalidation safety. -The initial `C0` read and later fill are not atomic. The fill is a normal overwrite, not a compare-and-set or write-if-still-missing operation: another writer can populate Redis after the semantic miss and be overwritten by the shadow fill. For tracked keys, the next atomic value-and-watermark read still rejects a frame whose client timestamp is at or before the watermark, so size `futureBufferMs` to cover the complete SoT, serialization, client queue, network, write interval, and fleet clock-skew budget when stale-serving protection matters. An untracked shadow fill has no such read fence and retains the ordinary TTL-based last-writer-wins contract; because it is detached, an older accepted source value may be written after a concurrent source mutation and remain until expiry. Shadow mode never repairs a frame returned as a non-null semantic `C0`; a physical frame rejected by normal tracked-read semantics is a miss and may be overwritten with a fresh TTL. Shadow work never invalidates, evicts local state, or changes the value returned to the caller. +The initial `C0` read and later fill are not atomic. A typed miss can suppress a fill already known to be fenced by the `C0` watermark, but an allowed fill is still a normal overwrite, not a compare-and-set or write-if-still-missing operation: another writer can populate Redis after the semantic miss and be overwritten by the shadow fill, or a later invalidation can fence it. For tracked keys, the next atomic value-and-watermark read still rejects a frame whose client timestamp is at or before the watermark, so size `futureBufferMs` to cover the complete SoT, serialization, client queue, network, write interval, and fleet clock-skew budget when stale-serving protection matters. An untracked shadow fill has no such read fence and retains the ordinary TTL-based last-writer-wins contract; because it is detached, an older accepted source value may be written after a concurrent source mutation and remain until expiry. Shadow mode never repairs a frame returned as a non-null semantic `C0`; a physical frame rejected by normal tracked-read semantics is a miss and may be overwritten with a fresh TTL unless its typed fence still rejects the candidate. Shadow work never invalidates, evicts local state, or changes the value returned to the caller. A served-hit sample invokes the wrapped function or inline loader as an additional source read, so that loader must be safe to call for observation. A ramped-down sample reuses the caller's ordinary invocation and does not add another SoT call. -For valid policies, shadow-specific source calls, cache-path Redis traffic, returned values, and metrics are unchanged when `shadow` is omitted or `shadow.ramp` is `0`. Shadow policy is grouped under `DialCacheKeyConfig.shadow`; consumers of the former flat ramp field must migrate to `shadow: { ramp }`. The public constructor, static defaults, and runtime provider results reject the removed field. `DialCacheKeyConfig.disabled()` explicitly disables the shadow ramp and mismatch logging. `shadowComparator` remains a typed `cached()` / `getOrLoad()` option because it defines stable use-case equality, while `shadowMaxInFlight` remains a per-instance concurrency limit. The semantic-miss bootstrap adds no additional ramp knob, Redis protocol operation, metric instrument, or label key; enabling shadowing authorizes the same Redis write described above. Untracked keys participate when they have a nonzero effective shadow ramp and an observable metrics hook. Exported unions include `remote_shadow` in `MetricLayer` and `superseded`, `filled`, `fill_error`, `redis_error`, and `confirmation_error` in `ShadowValidationOutcome`. TypeScript consumers with exhaustive switches or `Record` values must include those cases, and dashboards restricted to `layer="remote"` intentionally exclude detached traffic. +For valid policies, shadow-specific source calls, cache-path Redis traffic, returned values, and metrics are unchanged when `shadow` is omitted or `shadow.ramp` is `0`. Shadow policy is grouped under `DialCacheKeyConfig.shadow`; consumers of the former flat ramp field must migrate to `shadow: { ramp }`. The public constructor, static defaults, and runtime provider results reject the removed field. `DialCacheKeyConfig.disabled()` explicitly disables the shadow ramp and mismatch logging. `shadowComparator` remains a typed `cached()` / `getOrLoad()` option because it defines stable use-case equality, while `shadowMaxInFlight` remains a per-instance concurrency limit. The semantic-miss bootstrap adds no additional ramp knob, Redis protocol operation, metric instrument, or label key; enabling shadowing authorizes the same Redis write described above. Untracked keys participate when they have a nonzero effective shadow ramp and an observable metrics hook. Exported unions include `remote_shadow` in `MetricLayer` and `superseded`, `filled`, `fill_fenced`, `fill_error`, `redis_error`, and `confirmation_error` in `ShadowValidationOutcome`. TypeScript consumers with exhaustive switches or `Record` values must include those cases, and dashboards restricted to `layer="remote"` intentionally exclude detached traffic. ## Cached-value ownership @@ -721,7 +721,7 @@ Invalidation writes a Redis watermark at `{encodedNamespace:encodedKeyType:encod The internal `:dialcache-frame-v1` suffix identifies values written with DialCache's binary protocol. Watermarks are stored as decimal timestamps. -A cached Redis value whose writer-provided `createdAtMs` is older than or equal to the watermark is a tracked miss. `invalidateRemote(keyType, id, futureBufferMs)` proposes the invalidating process's `Date.now()` plus the buffer, and Lua keeps the greater of that proposal and the existing watermark. A tracked read obtains value and watermark in one primary-routed `MGET`; a missing watermark is the zero baseline, malformed or out-of-range decimal state fails closed, and `createdAtMs <= watermark` misses. Redis also returns `nil` for a wrong-type member of `MGET`, so a wrong-type watermark has the same zero-baseline behavior as an absent one until the next explicit invalidation repairs it. Native `MGET` must transfer the full stored frame before Node can apply the fence verdict, so a future window can repeatedly transfer a large fenced payload. Fallback still returns normally and writes one complete replacement frame even when that frame remains behind the watermark. A tracked invocation that reaches the Redis read/write path does not publish its fallback directly to process-local cache; a later validated Redis hit may warm it. Local-only, remote-policy-disabled, and ramped-down paths remain governed by local policy, while request-local memoization remains unconditional. A ramped-out invocation without shadow work does not consult Redis; selected tracked shadow reads remain watermark-aware. +A cached Redis value whose writer-provided `createdAtMs` is older than or equal to the watermark is a tracked miss. `invalidateRemote(keyType, id, futureBufferMs)` proposes the invalidating process's `Date.now()` plus the buffer, and Lua keeps the greater of that proposal and the existing watermark. A tracked read obtains value and watermark in one primary-routed `MGET`; a missing watermark is the zero baseline, malformed or out-of-range decimal state fails closed, and `createdAtMs <= watermark` misses. Redis also returns `nil` for a wrong-type member of `MGET`, so a wrong-type watermark has the same zero-baseline behavior as an absent one until the next explicit invalidation repairs it. When the same snapshot contains a present valid numeric watermark and an adapter-level semantic miss, the bundled adapter returns `RedisWatermarkMiss { observedWatermarkMs }`. After the fallback succeeds, DialCache samples one candidate timestamp before serialization. If `candidateCreatedAtMs <= observedWatermarkMs`, fallback still returns normally but the known-fenced replacement is skipped before serializer/compression/frame work and `SET`; if it is greater, the exact candidate is encoded in the normal complete-frame write. Missing or malformed watermark metadata and legacy custom adapters that return `null` retain the normal refill behavior. Native `MGET` must still transfer the full stored frame before Node can apply the fence verdict, so a future window can repeatedly transfer a large fenced payload even when replacement work is suppressed. A tracked invocation that reaches the Redis read/write path does not publish its fallback directly to process-local cache; a later validated Redis hit may warm it. Local-only, remote-policy-disabled, and ramped-down paths remain governed by local policy, while request-local memoization remains unconditional. A ramped-out invocation without shadow work does not consult Redis; selected tracked shadow reads remain watermark-aware. All serving timestamps come from application-process epoch clocks; DialCache does not call Redis `TIME`, estimate an offset, or compensate for skew. Participating application nodes therefore need external clock synchronization and monitoring. Healthy managed node pools commonly stay close, but Kubernetes does not guarantee a maximum offset, and pauses or NTP faults can be much larger than normal millisecond-scale skew. Relative clock differences shift logical expiry early or late, while frames dated after a reader clock fail closed until that clock catches up. Operation durations and deadlines continue to use the monotonic `performance.now()` clock. @@ -731,13 +731,13 @@ Tracked Redis value TTLs are capped at one hour; a dispatched write configured a `futureBufferMs` must be a nonnegative safe integer no greater than 31,536,000,000 (a fixed 365-day duration). The default is zero, but zero provides no stale-serving protection once a writer timestamp advances past the watermark. Every production invalidation should pass a named, application-owned nonzero value based on that application's measured or conservatively bounded timings; there is no universally safe library value. -Let `Dmax` be the maximum elapsed time from invalidation sampling until a stale pre-mutation `SET` can become visible in Redis, `S` the maximum writer-clock lead over the invalidator across participating application nodes, and `M` an operational margin. Callers that require stale-serving protection must satisfy `futureBufferMs >= Dmax + S + M`. Bound `Dmax` through source visibility or replication lag, the remaining tail of any fallback that may observe the pre-mutation value, `serializer.dump`, Redis client queueing or reconnect/offline-queue delay, network transit, and Redis execution and visibility. An unbounded offline queue or retry path makes a finite `Dmax` impossible. Invalidate only after the source mutation commits. The dangerous skew direction is a fast writer and slow invalidator: an undersized buffer can let delayed stale work receive a timestamp above the watermark and remain readable until expiry or another invalidation. The reverse direction is conservative. Overestimating the buffer increases fallback load, full-payload `MGET` transfer, and full-frame `SET`, replication, and AOF churn on hot large-value keys. It does not delay or suppress returning fallback values to callers. +Let `Dmax` be the maximum elapsed time from invalidation sampling until a stale pre-mutation `SET` can become visible in Redis, `S` the maximum writer-clock lead over the invalidator across participating application nodes, and `M` an operational margin. Callers that require stale-serving protection must satisfy `futureBufferMs >= Dmax + S + M`. Bound `Dmax` through source visibility or replication lag, the remaining tail of any fallback that may observe the pre-mutation value, `serializer.dump`, Redis client queueing or reconnect/offline-queue delay, network transit, and Redis execution and visibility. An unbounded offline queue or retry path makes a finite `Dmax` impossible. Invalidate only after the source mutation commits. The dangerous skew direction is a fast writer and slow invalidator: an undersized buffer can let delayed stale work receive a timestamp above the watermark and remain readable until expiry or another invalidation. The reverse direction is conservative. Overestimating the buffer still increases fallback load and full-payload `MGET` transfer on hot large-value keys. While a candidate remains behind a valid observed watermark, bundled and correctly opted-in adapters now avoid serializer/compression work plus full-frame `SET`, replication, and AOF churn; that work resumes once the candidate advances beyond the observed fence, and legacy `null` adapters continue the normal write path. The optimization does not delay or suppress returning fallback values to callers. The fixed one-minute watermark-TTL margin is retention slack after the covered visibility bound; it is not a substitute for `Dmax`. The operational contract should include the full post-sample dispatch tail in `Dmax` rather than relying on that slack. -This is a read-time timing contract rather than a cancellation or acquisition fence. The buffer makes covered frames unreadable; it does not stop their SETs, cancel in-flight work, or force fallback to read from an authoritative source. +This is a read-time timing contract rather than a cancellation or acquisition fence. The buffer makes covered frames unreadable. A typed miss can locally avoid a replacement already known to be fenced, but it does not cancel previously dispatched work, prevent a later watermark advance from fencing an allowed `SET`, or force fallback to read from an authoritative source. -The version-1 value envelope, Redis keys, and decimal watermarks remain wire-format-compatible. The adapter contract and Lua surface are not source-compatible: `write()` is now void and has no `watermarkKey`; the stamp script, placeholder helpers, and `DialCacheRedisPlaceholderLostError` are removed; `dialcacheRedisScripts` and `DialCacheNodeRedisScripts` are removed because node-redis manages invalidation dispatch internally; `ValkeyGlideRuntime` no longer requires `ClusterBatch`; invalidation is the only script; `fill_blocked` is removed from `ShadowValidationOutcome`; and `tracked_ttl_clamped` is added to `MetricErrorKind`, so exhaustive switches and `Record` values must add it. The Prometheus future-timestamp histogram now uses a dedicated skew-oriented bucket schema, which is incompatible with an existing same-name collector registered with the former default buckets. Follow the externally gated [protocol cutover](#protocol-cutover) before deployment. Deploy the future-timestamp metric and external node-clock alerts first. The metric is only a workload-shaped smoke detector: it cannot detect co-skewed readers and writers, an ahead invalidator, watermark skew hidden by a fenced miss, or which node is wrong. +The version-1 value envelope, Redis keys, and decimal watermarks remain wire-format-compatible. The adapter contract and Lua surface are not source-compatible: `write()` is now void and has no `watermarkKey`; the stamp script, placeholder helpers, and `DialCacheRedisPlaceholderLostError` are removed; `dialcacheRedisScripts` and `DialCacheNodeRedisScripts` are removed because node-redis manages invalidation dispatch internally; `ValkeyGlideRuntime` no longer requires `ClusterBatch`; invalidation is the only script; `fill_blocked` is removed from `ShadowValidationOutcome`; and `tracked_ttl_clamped` is added to `MetricErrorKind`, so exhaustive switches and `Record` values must add it. The conditional-refill addition itself does not change the Redis wire format, key format, Lua surface, command types, or round-trip shape; it may suppress an otherwise-dispatched `SET`. `RedisReadResult` is widened with opt-in `RedisWatermarkMiss`, but a custom adapter returning the legacy `DecodedRedisFrame | null` remains correct. `RedisWriteRequest.createdAtMs` is optional for source compatibility and direct callers may omit it; adapters returning typed watermark misses must honor a supplied value exactly. `fill_fenced` is added to `ShadowValidationOutcome`, so exhaustive switches and `Record` values must include it. The Prometheus future-timestamp histogram now uses a dedicated skew-oriented bucket schema, which is incompatible with an existing same-name collector registered with the former default buckets. Follow the externally gated [protocol cutover](#protocol-cutover) before deployment. Deploy the future-timestamp metric and external node-clock alerts first. The metric is only a workload-shaped smoke detector: it cannot detect co-skewed readers and writers, an ahead invalidator, watermark skew hidden by a fenced miss, or which node is wrong. Targeted invalidation is remote-only and enforced by Redis watermarks. `invalidateRemote` does not evict existing request-local or process-local entries. Strongly invalidated mutable data should disable request-local and process-local caching (or use a very short process-local TTL only when stale reads are acceptable). diff --git a/scripts/test-package.mjs b/scripts/test-package.mjs index bc57389..6e4c0db 100644 --- a/scripts/test-package.mjs +++ b/scripts/test-package.mjs @@ -149,6 +149,8 @@ const rootConsumer = `import { type RedisConfig, type RedisInvalidationRequest, type RedisReadContext, + type RedisReadResult, + type RedisWatermarkMiss, type RedisWriteRequest, type Serializer, type ShadowComparator, @@ -172,6 +174,7 @@ import { ceilSupportedCacheTtlMs, decodeRedisFrame, decodeTrackedRedisFrame, + decodeTrackedRedisReadResult, encodeRedisFrame, validateRedisScriptInvalidationReply, validateRedisSetReply, @@ -257,6 +260,7 @@ const shadowOutcomes: Readonly> = { mismatch: true, superseded: true, filled: true, + fill_fenced: true, fill_error: true, redis_error: true, source_error: true, @@ -339,6 +343,18 @@ const decodedStaleRedisFrame: DecodedRedisFrame | null = decodeTrackedRedisFrame emptyRedisFrame, Buffer.from("1"), ); +const decodedTrackedRedisReadResult: RedisReadResult = decodeTrackedRedisReadResult( + emptyRedisFrame, + Buffer.from("1"), +); +if ( + decodedTrackedRedisReadResult !== null + && "observedWatermarkMs" in decodedTrackedRedisReadResult +) { + const typedWatermarkMiss: RedisWatermarkMiss = decodedTrackedRedisReadResult; + const observedWatermarkMs: number = typedWatermarkMiss.observedWatermarkMs; + void observedWatermarkMs; +} const zeroTimestampRedisFrame: Buffer = encodeRedisFrame("pending", 0); const setReplyValidation: void = validateRedisSetReply("OK"); const invalidationReplyValidation: 1 = validateRedisScriptInvalidationReply(1); @@ -557,8 +573,11 @@ const compressionOperationMetricLabels: CompressionOperationMetricLabels = { const unboundedCompressionOutcome: CompressionOutcome = "inflated"; const customRedisClient: DialCacheRedisClient = { - // The optional second read argument preserves one-argument custom clients. - read: async () => ({ payload: Buffer.from([0, 255]), createdAtMs: 1 }), + // The optional second argument and widened result preserve legacy frame-or-null clients. + read: async (): Promise => ({ + payload: Buffer.from([0, 255]), + createdAtMs: 1, + }), write: async ({ value }) => { void (typeof value === "string" || Buffer.isBuffer(value)); }, @@ -581,9 +600,9 @@ const writeHasNoWatermark: "watermarkKey" extends keyof RedisWriteRequest ? fals const trackedWriteHasNoWatermarkTtlFloor: "watermarkTtlFloorMs" extends keyof RedisWriteRequest ? false : true = true; -const trackedWriteHasNoCreatedAt: "createdAtMs" extends keyof RedisWriteRequest - ? false - : true = true; +const writeAcceptsOptionalCreatedAt: {} extends Pick + ? true + : false = true; const invalidationHasNoWatermarkTtlFloor: "watermarkTtlFloorMs" extends keyof RedisInvalidationRequest ? false : true = true; @@ -597,6 +616,17 @@ const legacyTrackedWriteRequest: RedisWriteRequest = { cacheTtlMs: 1_000, value: "tracked", }; +const timestampedWriteRequest: RedisWriteRequest = { + valueKey: "tracked:{id}:value", + cacheTtlMs: 1_000, + value: "tracked", + createdAtMs: 1, +}; +const omittedTimestampWriteRequest: RedisWriteRequest = { + valueKey: "legacy:{id}:value", + cacheTtlMs: 1_000, + value: "legacy", +}; const legacyInvalidationRequest: RedisInvalidationRequest = { watermarkKey: "tracked:{id}:watermark", futureBufferMs: 0, @@ -731,10 +761,12 @@ void cacheHasNoClose; void clientHasNoFlushAll; void writeHasNoWatermark; void trackedWriteHasNoWatermarkTtlFloor; -void trackedWriteHasNoCreatedAt; +void writeAcceptsOptionalCreatedAt; void invalidationHasNoWatermarkTtlFloor; void invalidationHasNoInvalidatedAt; void legacyTrackedWriteRequest; +void timestampedWriteRequest; +void omittedTimestampWriteRequest; void legacyInvalidationRequest; void configHasNoMetricsRegistry; void configHasNoMetricsPrefix; @@ -974,9 +1006,19 @@ if (esmRoundTrip?.payload !== "value" || esmRoundTrip.createdAtMs !== 1) { if (redisProtocol.decodeTrackedRedisFrame(redisProtocol.encodeRedisFrame("pending", 0), Buffer.from("0")) !== null) { throw new Error("The packed ESM tracked decoder did not fence an equal timestamp"); } +const esmWatermarkMiss = redisProtocol.decodeTrackedRedisReadResult( + redisProtocol.encodeRedisFrame("pending", 0), + Buffer.from("0"), +); +if (esmWatermarkMiss?.observedWatermarkMs !== 0 || "payload" in esmWatermarkMiss) { + throw new Error("The packed ESM tracked result decoder did not preserve the observed watermark miss"); +} if (redisProtocol.decodeTrackedRedisFrame(redisProtocol.encodeRedisFrame("value", 1), null)?.payload !== "value") { throw new Error("The packed ESM tracked decoder did not use zero for a missing watermark"); } +if (redisProtocol.decodeTrackedRedisReadResult(null, null) !== null) { + throw new Error("The packed ESM tracked result decoder did not preserve a generic miss without a watermark"); +} if ( "REDIS_FRAME_VERSION" in redisProtocol || "REDIS_ENCODING_UTF8" in redisProtocol @@ -1349,9 +1391,19 @@ if (cjsRoundTrip?.payload !== "value" || cjsRoundTrip.createdAtMs !== 1) { if (redisProtocol.decodeTrackedRedisFrame(redisProtocol.encodeRedisFrame("pending", 0), Buffer.from("0")) !== null) { throw new Error("The packed CommonJS tracked decoder did not fence an equal timestamp"); } +const cjsWatermarkMiss = redisProtocol.decodeTrackedRedisReadResult( + redisProtocol.encodeRedisFrame("pending", 0), + Buffer.from("0"), +); +if (cjsWatermarkMiss?.observedWatermarkMs !== 0 || "payload" in cjsWatermarkMiss) { + throw new Error("The packed CommonJS tracked result decoder did not preserve the observed watermark miss"); +} if (redisProtocol.decodeTrackedRedisFrame(redisProtocol.encodeRedisFrame("value", 1), null)?.payload !== "value") { throw new Error("The packed CommonJS tracked decoder did not use zero for a missing watermark"); } +if (redisProtocol.decodeTrackedRedisReadResult(null, null) !== null) { + throw new Error("The packed CommonJS tracked result decoder did not preserve a generic miss without a watermark"); +} if ( "REDIS_FRAME_VERSION" in redisProtocol || "REDIS_ENCODING_UTF8" in redisProtocol @@ -1520,6 +1572,7 @@ const redisProtocol = await import("dialcache/redis-protocol"); await import("dialcache/node-redis"); ${packedInvalidationCheckSource} const esmCreatedAtMs = 1700000000123; +const esmAdapterClockMs = esmCreatedAtMs + 999; if (appGlide.Script === otherGlide.Script) { throw new Error("The package test requires two distinct GLIDE module instances"); } @@ -1552,12 +1605,13 @@ const esmGlideRuntime = { }; const adapter = glide.createValkeyGlideDialCacheClient(esmFakeGlideClient, esmGlideRuntime); const esmNativeDateNow = Date.now; -Date.now = () => esmCreatedAtMs; +Date.now = () => esmAdapterClockMs; try { await adapter.write({ valueKey: "tracked:{id}:value", cacheTtlMs: 1_000, value: "payload", + createdAtMs: esmCreatedAtMs, }); if ( esmWriteCommand[0] !== "SET" @@ -1567,7 +1621,7 @@ try { || esmWriteCommand[2][0] !== 1 || esmWriteCommand[2].readBigUInt64BE(1) !== BigInt(esmCreatedAtMs) ) { - throw new Error("The packed ESM GLIDE write did not send one complete client-stamped frame"); + throw new Error("The packed ESM GLIDE write did not preserve the supplied frame timestamp exactly"); } const trackedRead = await adapter.read({ valueKey: "tracked:{id}:value", @@ -1649,7 +1703,7 @@ void (async () => { || cjsWriteCommand[2][0] !== 1 || cjsWriteCommand[2].readBigUInt64BE(1) !== BigInt(cjsCreatedAtMs) ) { - throw new Error("The packed CommonJS GLIDE write did not send one complete client-stamped frame"); + throw new Error("The packed CommonJS GLIDE write did not stamp an omitted timestamp from its client clock"); } const trackedRead = await adapter.read({ valueKey: "tracked:{id}:value", diff --git a/src/dialcache.ts b/src/dialcache.ts index f2abe94..ea0938c 100644 --- a/src/dialcache.ts +++ b/src/dialcache.ts @@ -24,7 +24,12 @@ import { type MetricLayer, type ShadowValidationOutcome, } from "./metrics.js"; -import type { DecodedRedisFrame, RedisCachePayload } from "./redis-client.js"; +import type { + DecodedRedisFrame, + RedisCachePayload, + RedisReadResult, + RedisWatermarkMiss, +} from "./redis-client.js"; import type { Serializer } from "./serializer.js"; import type { CacheGetResult, RemoteCacheGetResult } from "./internal/cache-result.js"; import { MAX_TIMER_DELAY_MS, withMonotonicDeadline } from "./internal/deadline.js"; @@ -866,7 +871,12 @@ export class DialCache { || (remoteWriteConfig !== undefined && key.trackForInvalidation); if (remoteWriteConfig !== undefined) { try { - await redisCache.put(key, value, remoteWriteConfig); + await redisCache.put( + key, + value, + remoteWriteConfig, + remote.status === "miss" ? remote.watermarkMiss : undefined, + ); } catch (error) { this.logger.warn("Error putting value in Redis cache", error); } @@ -1044,7 +1054,7 @@ export class DialCache { const readShadowFrame = ( maxAgeSec: number | null, futureFramePolicy: FutureFramePolicy, - ): Promise => { + ): Promise => { const read = redisCache.startPayloadReadForShadow( key, maxAgeSec, @@ -1089,20 +1099,24 @@ export class DialCache { } let shadowFillConfig: ResolvedRemoteLayerConfig | null = null; + let shadowFillWatermarkMiss: RedisWatermarkMiss | undefined; if (start.kind === "redis") { - let frame: DecodedRedisFrame | null; + let readResult: RedisReadResult; try { - frame = await readShadowFrame(start.remoteConfig.ttlSec, "reject"); + readResult = await readShadowFrame(start.remoteConfig.ttlSec, "reject"); } catch { return "redis_error"; } if (abandonIfExpired()) { return "timeout"; } - if (frame === null) { + if (isRedisWatermarkMiss(readResult)) { + shadowFillConfig = start.remoteConfig; + shadowFillWatermarkMiss = readResult; + } else if (readResult === null) { shadowFillConfig = start.remoteConfig; } else { - flight.cachedFrame = frame; + flight.cachedFrame = readResult; } } @@ -1134,18 +1148,19 @@ export class DialCache { if (shadowFillConfig !== null) { try { - await redisCache.putForShadow( + const filled = await redisCache.putForShadow( key, sourceValue, shadowFillConfig, () => !abandonIfExpired(), + shadowFillWatermarkMiss, ); // A late result remains the already-emitted whole-job timeout: // dispatch success does not retroactively change its outcome. if (abandonIfExpired()) { return "timeout"; } - return "filled"; + return filled ? "filled" : "fill_fenced"; } catch (error) { this.logger.warn("Error populating Redis from DialCache shadow work", error); return "fill_error"; @@ -1189,9 +1204,9 @@ export class DialCache { return "match"; } - let confirmationFrame: DecodedRedisFrame | null; + let confirmationResult: RedisReadResult; try { - confirmationFrame = await readShadowFrame(null, "retain"); + confirmationResult = await readShadowFrame(null, "retain"); } catch { return "confirmation_error"; } @@ -1203,6 +1218,9 @@ export class DialCache { if (originalFrame === null) { return "timeout"; } + const confirmationFrame = isRedisWatermarkMiss(confirmationResult) + ? null + : confirmationResult; if (confirmationFrame === null || !redisPayloadsEqual(originalFrame.payload, confirmationFrame.payload)) { return "superseded"; } @@ -1755,6 +1773,14 @@ function redisPayloadsEqual(left: RedisCachePayload, right: RedisCachePayload): return Buffer.isBuffer(right) && right.equals(Buffer.from(left, "utf8")); } +function isRedisWatermarkMiss(result: RedisReadResult): result is RedisWatermarkMiss { + return typeof result === "object" + && result !== null + && "observedWatermarkMs" in result + && !("payload" in result) + && !("createdAtMs" in result); +} + async function settleUnexpectedThenable(value: unknown): Promise { if (value === null || (typeof value !== "object" && typeof value !== "function")) { return; diff --git a/src/index.ts b/src/index.ts index 7a8d269..9203d24 100644 --- a/src/index.ts +++ b/src/index.ts @@ -62,6 +62,8 @@ export type { RedisInvalidationRequest, RedisReadContext, RedisReadRequest, + RedisReadResult, + RedisWatermarkMiss, RedisWriteRequest, } from "./redis-client.js"; export { JsonSerializer } from "./serializer.js"; diff --git a/src/internal/cache-result.ts b/src/internal/cache-result.ts index df53ca7..9b987f5 100644 --- a/src/internal/cache-result.ts +++ b/src/internal/cache-result.ts @@ -1,6 +1,6 @@ import type { ResolvedLayerConfig, ResolvedRemoteLayerConfig } from "./runtime-config.js"; import type { DisabledReason } from "../metrics.js"; -import type { DecodedRedisFrame } from "../redis-client.js"; +import type { DecodedRedisFrame, RedisWatermarkMiss } from "../redis-client.js"; export type CacheGetResult = | { readonly status: "hit"; readonly value: T } @@ -25,6 +25,7 @@ export type RedisCacheGetResult = readonly status: "miss"; readonly config: ResolvedRemoteLayerConfig; readonly reason: RedisCacheMissReason; + readonly watermarkMiss?: RedisWatermarkMiss; }; export type RemoteCacheGetResult = diff --git a/src/internal/redis-cache.ts b/src/internal/redis-cache.ts index 0ff271e..cf0e0c0 100644 --- a/src/internal/redis-cache.ts +++ b/src/internal/redis-cache.ts @@ -11,7 +11,13 @@ import { type MetricLayer, type StaleRecoveryOutcome, } from "../metrics.js"; -import type { DecodedRedisFrame, DialCacheRedisClient, RedisCachePayload } from "../redis-client.js"; +import type { + DecodedRedisFrame, + DialCacheRedisClient, + RedisCachePayload, + RedisReadResult, + RedisWatermarkMiss, +} from "../redis-client.js"; import { JsonSerializer, type Serializer } from "../serializer.js"; import type { RedisCacheGetResult } from "./cache-result.js"; import { @@ -23,6 +29,7 @@ import { } from "./compression.js"; import { assertValidDeadlineMs, withMonotonicDeadline } from "./deadline.js"; import { cacheTtlSecToMs, MAX_TRACKED_REDIS_VALUE_TTL_MS } from "./duration.js"; +import { assertValidRedisTimestampMs } from "./redis-payload.js"; import type { ResolvedRemoteLayerConfig } from "./runtime-config.js"; export interface RedisConfig { @@ -53,7 +60,7 @@ interface RedisCacheOptions { interface StartedRedisRead { /** Result bounded by the effective Redis read deadline. */ - readonly result: Promise; + readonly result: Promise; /** Fulfills only after the underlying semantic Redis read settles. */ readonly settled: Promise; } @@ -121,9 +128,9 @@ export class RedisCache { const start = performance.now(); this.recordMetric((metrics) => metrics.request(labelsFor(key, metricLayer))); try { - let frame: DecodedRedisFrame | null; + let result: RedisReadResult; try { - frame = await this.startRawPayloadRead( + result = await this.startRawPayloadRead( key, readTimeoutMs, false, @@ -136,7 +143,16 @@ export class RedisCache { ); throw error; } - if (frame === null) { + if (isRedisWatermarkMiss(result)) { + this.recordMetric((metrics) => metrics.miss(labelsFor(key, metricLayer))); + return { + status: "miss", + config: layerConfig, + reason: "cache_miss", + watermarkMiss: result, + }; + } + if (result === null) { this.recordMetric((metrics) => metrics.miss(labelsFor(key, metricLayer))); return { status: "miss", config: layerConfig, reason: "cache_miss" }; } @@ -144,19 +160,19 @@ export class RedisCache { // Classify the raw value/watermark snapshot from one application-clock // sample after the bounded read settles. M is the absolute ceiling and F // remains the ordinary serving boundary. - const frameAge = this.frameAge(key, frame, metricLayer); + const frameAge = this.frameAge(key, result, metricLayer); if (frameAge.status !== "valid" || frameAge.ageMs >= maximumAgeMs) { this.recordMetric((metrics) => metrics.miss(labelsFor(key, metricLayer))); return { status: "miss", config: layerConfig, reason: "cache_miss" }; } if (frameAge.ageMs >= freshAgeMs) { this.recordMetric((metrics) => metrics.miss(labelsFor(key, metricLayer))); - return { status: "retained", config: layerConfig, frame }; + return { status: "retained", config: layerConfig, frame: result }; } try { - const value = await this.deserializePayload(key, frame.payload, metricLayer); - return { status: "hit", value, frame }; + const value = await this.deserializePayload(key, result.payload, metricLayer); + return { status: "hit", value, frame: result }; } catch { this.recordMetric((metrics) => metrics.miss(labelsFor(key, metricLayer))); return { status: "miss", config: layerConfig, reason: "deserialization_error" }; @@ -238,8 +254,20 @@ export class RedisCache { ); } - async put(key: DialCacheKey, value: T, config: ResolvedRemoteLayerConfig): Promise { - await this.putWithLayer(key, value, retentionTtlSecFor(config), CacheLayer.REMOTE); + async put( + key: DialCacheKey, + value: T, + config: ResolvedRemoteLayerConfig, + watermarkMiss?: RedisWatermarkMiss, + ): Promise { + await this.putWithLayer( + key, + value, + retentionTtlSecFor(config), + CacheLayer.REMOTE, + undefined, + watermarkMiss, + ); } /** Populate a detached Redis miss using the caller's resolved policy snapshot. */ @@ -248,13 +276,15 @@ export class RedisCache { value: T, config: ResolvedRemoteLayerConfig, shouldWrite: () => boolean, - ): Promise { - await this.putWithLayer( + watermarkMiss?: RedisWatermarkMiss, + ): Promise { + return await this.putWithLayer( key, value, retentionTtlSecFor(config), REMOTE_SHADOW_CACHE_LAYER, shouldWrite, + watermarkMiss, ); } @@ -264,11 +294,25 @@ export class RedisCache { ttlSec: number, metricLayer: MetricLayer, shouldWrite?: () => boolean, - ): Promise { + watermarkMiss?: RedisWatermarkMiss, + ): Promise { const configuredTtlMs = cacheTtlSecToMs(ttlSec); const cacheTtlMs = key.trackForInvalidation ? Math.min(configuredTtlMs, MAX_TRACKED_REDIS_VALUE_TTL_MS) : configuredTtlMs; + let createdAtMs: number | undefined; + if (key.trackForInvalidation && watermarkMiss !== undefined) { + createdAtMs = Date.now(); + try { + assertValidRedisTimestampMs(createdAtMs); + } catch (error) { + this.recordError(key, metricLayer, "cache_write"); + throw error; + } + if (createdAtMs <= watermarkMiss.observedWatermarkMs) { + return false; + } + } const start = performance.now(); let serialized: string | Buffer; @@ -307,7 +351,7 @@ export class RedisCache { } this.recordMetric((metrics) => metrics.observeStoredSize?.(labelsFor(key, metricLayer), payloadSize(serialized))); if (shouldWrite !== undefined && !shouldWrite()) { - return; + return false; } if (cacheTtlMs < configuredTtlMs) { this.recordError(key, metricLayer, "tracked_ttl_clamped"); @@ -318,11 +362,13 @@ export class RedisCache { valueKey: this.redisKey(key), cacheTtlMs, value: serialized, + ...(createdAtMs === undefined ? {} : { createdAtMs }), }); } catch (error) { this.recordError(key, metricLayer, "cache_write"); throw error; } + return true; } async invalidate(keyType: string, id: string, futureBufferMs = 0, namespace = "urn"): Promise { @@ -383,7 +429,7 @@ export class RedisCache { unrefTimer, }); return { - result: bounded, + result: bounded.then((result) => this.validateReadResult(key, result)), settled: pending.then( () => undefined, () => undefined, @@ -410,11 +456,11 @@ export class RedisCache { futureFramePolicy, ); const result = read.result.then( - (frame) => { - if (frame === null) { + (result) => { + if (result === null || isRedisWatermarkMiss(result)) { this.recordMetric((metrics) => metrics.miss(labelsFor(key, metricLayer))); } - return frame; + return result; }, (error: unknown) => { this.recordError( @@ -432,22 +478,35 @@ export class RedisCache { private validateFrameAge( key: DialCacheKey, - frame: DecodedRedisFrame | null, + result: RedisReadResult, maxAgeMs: number | null, metricLayer: MetricLayer, futureFramePolicy: FutureFramePolicy, - ): DecodedRedisFrame | null { - if (frame === null) { - return null; + ): RedisReadResult { + if (result === null || isRedisWatermarkMiss(result)) { + return result; } - const age = this.frameAge(key, frame, metricLayer); + const age = this.frameAge(key, result, metricLayer); if (age.status === "future") { - return futureFramePolicy === "reject" ? null : frame; + return futureFramePolicy === "reject" ? null : result; } if (age.status === "invalid") { return null; } - return maxAgeMs === null || age.ageMs < maxAgeMs ? frame : null; + return maxAgeMs === null || age.ageMs < maxAgeMs ? result : null; + } + + private validateReadResult(key: DialCacheKey, result: RedisReadResult): RedisReadResult { + try { + if (!isRedisWatermarkMiss(result)) { + return result; + } + return key.trackForInvalidation && isValidRedisWatermarkMiss(result) + ? result + : null; + } catch { + return null; + } } private frameAge( @@ -552,3 +611,16 @@ function payloadSize(payload: string | Buffer): number { function elapsedSeconds(startMs: number): number { return Math.max((performance.now() - startMs) / 1000, 0); } + +function isRedisWatermarkMiss(result: RedisReadResult): result is RedisWatermarkMiss { + return typeof result === "object" + && result !== null + && "observedWatermarkMs" in result + && !("payload" in result) + && !("createdAtMs" in result); +} + +function isValidRedisWatermarkMiss(miss: RedisWatermarkMiss): boolean { + return Number.isSafeInteger(miss.observedWatermarkMs) + && miss.observedWatermarkMs >= 0; +} diff --git a/src/internal/redis-payload.ts b/src/internal/redis-payload.ts index d77ab83..2cfd953 100644 --- a/src/internal/redis-payload.ts +++ b/src/internal/redis-payload.ts @@ -3,6 +3,8 @@ import { DialCacheRedisPayloadError, type DecodedRedisFrame, type RedisCachePayload, + type RedisReadResult, + type RedisWatermarkMiss, } from "../redis-client.js"; const REDIS_FRAME_VERSION = 1; @@ -105,35 +107,70 @@ export function decodeRedisFrame(raw: unknown): DecodedRedisFrame | null { } /** - * Decode a tracked DialCache frame against a watermark from the same atomic, - * authoritative snapshot into its serializer payload and header creation time - * (application time supplied by the writer). A missing watermark is the - * natural zero baseline; malformed state and frames created at or before the - * watermark are cache misses. Invalid runtime reply types and unsupported - * payload encodings throw typed errors. + * Decode a tracked DialCache read while preserving a trustworthy observed + * watermark for semantic misses. A present valid numeric watermark produces a + * `RedisWatermarkMiss` whenever the value is absent, unsupported, or fenced; + * missing or malformed watermark metadata retains the generic `null` miss. + * Invalid runtime reply types and unsupported payload encodings on otherwise + * eligible frames throw typed errors. + * + * Custom adapters opting into this result must also honor a supplied + * `RedisWriteRequest.createdAtMs` exactly. */ -export function decodeTrackedRedisFrame( +export function decodeTrackedRedisReadResult( raw: unknown, rawWatermark: unknown, -): DecodedRedisFrame | null { +): RedisReadResult { const frame = validateRedisBulkStringReply(raw); const watermarkFrame = validateRedisBulkStringReply(rawWatermark); - if (!isSupportedRedisFrame(frame)) { - return null; + if (watermarkFrame === null) { + return decodeTrackedFrame(frame, 0, null); } - const watermark = watermarkFrame === null ? 0 : parseRedisWatermark(watermarkFrame); + const watermark = parseRedisWatermark(watermarkFrame); if (watermark === null) { return null; } + return decodeTrackedFrame(frame, watermark, { observedWatermarkMs: watermark }); +} + +/** + * Backward-compatible tracked-frame decoder. It preserves the established + * `DecodedRedisFrame | null` surface by collapsing typed watermark misses. + * Use `decodeTrackedRedisReadResult` to opt a custom adapter into conditional + * fenced-refill suppression. + */ +export function decodeTrackedRedisFrame( + raw: unknown, + rawWatermark: unknown, +): DecodedRedisFrame | null { + const result = decodeTrackedRedisReadResult(raw, rawWatermark); + return isRedisWatermarkMiss(result) ? null : result; +} + +function decodeTrackedFrame( + frame: Buffer | null, + watermark: number, + miss: RedisWatermarkMiss | null, +): RedisReadResult { + if (!isSupportedRedisFrame(frame)) { + return miss; + } const createdAtMs = readFrameCreatedAtMs(frame); return createdAtMs <= watermark - ? null + ? miss : { payload: decodeRedisPayload(frame.subarray(REDIS_FRAME_HEADER_BYTES)), createdAtMs, }; } +function isRedisWatermarkMiss(result: RedisReadResult): result is RedisWatermarkMiss { + return result !== null + && "observedWatermarkMs" in result + && !("payload" in result) + && !("createdAtMs" in result); +} + function readFrameCreatedAtMs(frame: Buffer): number { return Number(frame.readBigUInt64BE(REDIS_FRAME_TIMESTAMP_OFFSET)); } diff --git a/src/metrics.ts b/src/metrics.ts index 925955d..5ba9af9 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -16,6 +16,7 @@ export type ShadowValidationOutcome = | "mismatch" | "superseded" | "filled" + | "fill_fenced" | "fill_error" | "redis_error" | "source_error" diff --git a/src/node-redis.ts b/src/node-redis.ts index c7f28d2..b88becc 100644 --- a/src/node-redis.ts +++ b/src/node-redis.ts @@ -8,7 +8,7 @@ import { import { assertValidRedisTimestampMs, decodeRedisFrame, - decodeTrackedRedisFrame, + decodeTrackedRedisReadResult, encodeRedisFrame, } from "./internal/redis-payload.js"; import { ceilSupportedCacheTtlMs } from "./internal/duration.js"; @@ -138,13 +138,14 @@ export function createNodeRedisDialCacheClient(client: NodeRedisClient): DialCac valueKey, watermarkKey, ); - return decodeTrackedRedisFrame(rawValue, rawWatermark); + return decodeTrackedRedisReadResult(rawValue, rawWatermark); }, async write(request) { const { valueKey, value } = request; const cacheTtlMs = ceilSupportedCacheTtlMs(request.cacheTtlMs); + const createdAtMs = request.createdAtMs === undefined ? Date.now() : request.createdAtMs; validateRedisSetReply( - await sendFrameSet(client, valueKey, encodeRedisFrame(value, Date.now()), cacheTtlMs), + await sendFrameSet(client, valueKey, encodeRedisFrame(value, createdAtMs), cacheTtlMs), ); }, async invalidate({ watermarkKey, futureBufferMs }) { diff --git a/src/redis-client.ts b/src/redis-client.ts index 780656d..bc5720b 100644 --- a/src/redis-client.ts +++ b/src/redis-client.ts @@ -80,6 +80,24 @@ export interface DecodedRedisFrame { readonly createdAtMs: number; } +/** + * A semantic tracked-read miss carrying a trustworthy write fence from the + * same authoritative value-and-watermark snapshot. A refill stamped at or + * before `observedWatermarkMs` is known to remain unreadable. + */ +export interface RedisWatermarkMiss { + readonly observedWatermarkMs: number; + readonly payload?: never; + readonly createdAtMs?: never; +} + +/** + * Semantic Redis read result. `null` remains the generic/legacy miss; bundled + * adapters return `RedisWatermarkMiss` only when a tracked read observed a + * present, valid numeric watermark that can safely fence a candidate refill. + */ +export type RedisReadResult = DecodedRedisFrame | RedisWatermarkMiss | null; + interface RedisValueRequest { readonly valueKey: string; } @@ -107,6 +125,15 @@ export interface RedisWriteRequest extends RedisValueRequest { /** Positive integer no greater than 31,536,000,000 (365 days). */ readonly cacheTtlMs: number; readonly value: RedisCachePayload; + /** + * Nonnegative safe-integer epoch milliseconds to encode in the frame. + * DialCache core supplies this for refills following `RedisWatermarkMiss`. + * It remains optional so ordinary refills, existing direct adapter callers, + * and custom adapter implementations keep their established behavior. An + * adapter that returns `RedisWatermarkMiss` must honor a supplied value + * exactly so the refill decision and stored frame cannot diverge. + */ + readonly createdAtMs?: number; } export interface RedisInvalidationRequest { @@ -133,9 +160,10 @@ export interface RedisInvalidationRequest { */ export interface DialCacheRedisClient { /** - * Read a DialCache Redis frame and return its decoded serializer payload - * together with the frame header's creation time. Implementations must use - * `decodeRedisFrame` / `decodeTrackedRedisFrame` from + * Read a DialCache Redis frame. Hits return the decoded serializer payload + * with the frame header's creation time. Implementations must use + * `decodeRedisFrame` and either `decodeTrackedRedisFrame` (legacy null + * misses) or `decodeTrackedRedisReadResult` (typed watermark misses) from * `dialcache/redis-protocol`, or preserve their exact behavior. * * Raw values are Redis bulk strings (`Buffer`) or null. A missing value, a @@ -150,22 +178,31 @@ export interface DialCacheRedisClient { * Tracked implementations must read the value and watermark atomically from * one authoritative snapshot; replica lag must not hide an invalidation. * - * A non-null frame is transferred to DialCache. A returned Buffer payload + * Implementations may return a `RedisWatermarkMiss` for a tracked semantic + * miss when the same snapshot contained a present, valid numeric watermark. + * Existing adapters may continue returning `null` and remain correct while + * missing the conditional refill optimization. Adapters that opt into the + * typed miss must also honor `RedisWriteRequest.createdAtMs` when supplied. + * + * A returned frame's payload is transferred to DialCache. A returned Buffer * must remain stable and must not be mutated, pooled, or reused after this * method settles; DialCache may retain it for source-error recovery or * best-effort shadow work. Adapters that recycle response storage must * return a dedicated Buffer. */ - read(request: RedisReadRequest, context?: RedisReadContext): Awaitable; + read(request: RedisReadRequest, context?: RedisReadContext): Awaitable; /** * Write a DialCache Redis frame using the `dialcache/redis-protocol` * encoders, or preserve their exact behavior. * - * All writes are one native `SET valueKey frame PX cacheTtlMs` whose - * frame comes from `encodeRedisFrame` with a client-clock `createdAtMs`. + * All writes are one native `SET valueKey frame PX cacheTtlMs` whose frame + * comes from `encodeRedisFrame`. Honor `request.createdAtMs` exactly when it + * is supplied; callers that omit it may be stamped from the adapter's client + * clock. * DialCache uses every frame's decoded `createdAtMs` for future-time - * rejection and logical-age enforcement, and for shadow value-age - * observations, so writers must stamp real client time, not a constant. + * rejection and logical-age enforcement, and for shadow and stale-recovery + * value-age observations, so writers must stamp real client time, not a + * constant. * * Tracked and untracked writes use the same complete-frame SET. Core caps a * tracked value's physical TTL at one hour. Under the documented clock-skew diff --git a/src/redis-protocol.ts b/src/redis-protocol.ts index d5ce8f9..d401e58 100644 --- a/src/redis-protocol.ts +++ b/src/redis-protocol.ts @@ -18,9 +18,14 @@ export { INVALIDATE_CACHE_SCRIPT } from "./internal/redis-scripts.js"; export { decodeRedisFrame, decodeTrackedRedisFrame, + decodeTrackedRedisReadResult, encodeRedisFrame, } from "./internal/redis-payload.js"; -export type { DecodedRedisFrame } from "./redis-client.js"; +export type { + DecodedRedisFrame, + RedisReadResult, + RedisWatermarkMiss, +} from "./redis-client.js"; export { validateRedisScriptInvalidationReply, validateRedisSetReply, diff --git a/src/valkey-glide.ts b/src/valkey-glide.ts index 1ad2c92..6bc1c08 100644 --- a/src/valkey-glide.ts +++ b/src/valkey-glide.ts @@ -7,7 +7,7 @@ import { import { assertValidRedisTimestampMs, decodeRedisFrame, - decodeTrackedRedisFrame, + decodeTrackedRedisReadResult, encodeRedisFrame, } from "./internal/redis-payload.js"; import { @@ -163,13 +163,14 @@ export function createValkeyGlideDialCacheClient( if (!Array.isArray(pair) || pair.length !== 2) { throw new DialCacheRedisPayloadError("Invalid DialCache Redis payload reply"); } - return decodeTrackedRedisFrame(pair[0], pair[1]); + return decodeTrackedRedisReadResult(pair[0], pair[1]); }, async write(request) { const { valueKey, value } = request; const cacheTtlMs = ceilSupportedCacheTtlMs(request.cacheTtlMs); const execOptions = keyedOptions(valueKey); - const frame = encodeRedisFrame(value, Date.now()); + const createdAtMs = request.createdAtMs === undefined ? Date.now() : request.createdAtMs; + const frame = encodeRedisFrame(value, createdAtMs); validateRedisSetReply( await client.customCommand(["SET", valueKey, frame, "PX", String(cacheTtlMs)], execOptions), ); diff --git a/test/datadog.test.ts b/test/datadog.test.ts index 3c92dbc..a0c5b1b 100644 --- a/test/datadog.test.ts +++ b/test/datadog.test.ts @@ -111,6 +111,7 @@ const SHADOW_VALIDATION_OUTCOMES: Readonly mismatch: true, superseded: true, filled: true, + fill_fenced: true, fill_error: true, redis_error: true, source_error: true, diff --git a/test/dialcache-compression.test.ts b/test/dialcache-compression.test.ts index 2e48ddb..718a0b4 100644 --- a/test/dialcache-compression.test.ts +++ b/test/dialcache-compression.test.ts @@ -1,7 +1,7 @@ import { randomBytes } from "node:crypto"; import { zstdCompressSync } from "node:zlib"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { CacheLayer, @@ -38,6 +38,7 @@ class RecordingMetrics implements DialCacheMetricsAdapter { readonly sizeCalls: Array<{ readonly labels: CacheMetricLabels; readonly bytes: number }> = []; readonly storedSizeCalls: Array<{ readonly labels: CacheMetricLabels; readonly bytes: number }> = []; readonly durationCalls: CompressionOperationMetricLabels[] = []; + readonly serializationCalls: SerializationMetricLabels[] = []; request(): void {} miss(): void {} @@ -46,7 +47,9 @@ class RecordingMetrics implements DialCacheMetricsAdapter { invalidation(): void {} observeGet(): void {} observeFallback(): void {} - observeSerialization(_labels: SerializationMetricLabels, _seconds: number): void {} + observeSerialization(labels: SerializationMetricLabels, _seconds: number): void { + this.serializationCalls.push(labels); + } compression(labels: CompressionMetricLabels): void { this.compressionCalls.push(labels); @@ -70,6 +73,49 @@ class RecordingMetrics implements DialCacheMetricsAdapter { } describe("DialCache Redis payload compression", () => { + it("skips a fenced large tracked refill before serialization and compression", async () => { + const redis = new FakeRedis(); + const key = new DialCacheKey({ + keyType: "user_id", + id: "123", + useCase: "CompressionFencedRefill", + trackForInvalidation: true, + }); + redis.setRaw(`${key.prefix}#watermark`, String(Date.now() + 60_000)); + const metrics = new RecordingMetrics(); + const dump = vi.fn((): string => { + throw new Error("fenced refill must not serialize"); + }); + const serializer: Serializer> = { + dump, + load: () => { + throw new Error("missing value must not deserialize"); + }, + }; + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const getUser = dialcache.cached(async (userId: string) => largeValue(userId), { + keyType: "user_id", + useCase: "CompressionFencedRefill", + cacheKey: (userId) => userId, + trackForInvalidation: true, + defaultConfig: remoteOnly(), + serializer, + }); + + const value = await dialcache.enable(async () => await getUser("123")); + + expect(value).toEqual(largeValue("123")); + expect(dump).not.toHaveBeenCalled(); + expect(redis.setCalls).toBe(0); + expect(redis.values.has(`${key.urn}:dialcache-frame-v1`)).toBe(false); + expect(metrics.serializationCalls).toEqual([]); + expect(metrics.compressionCalls).toEqual([]); + expect(metrics.durationCalls).toEqual([]); + expect(metrics.sizeCalls).toEqual([]); + expect(metrics.storedSizeCalls).toEqual([]); + expect(metrics.ratioCalls).toEqual([]); + }); + it("compresses large values transparently and reads them back across processes", async () => { const redis = new FakeRedis(); const writer = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); diff --git a/test/dialcache-invalidation.test.ts b/test/dialcache-invalidation.test.ts index 3523c8c..fc89ea3 100644 --- a/test/dialcache-invalidation.test.ts +++ b/test/dialcache-invalidation.test.ts @@ -20,7 +20,7 @@ import { MAX_TRACKED_REDIS_VALUE_TTL_MS, } from "../src/internal/duration.js"; import { MIN_WATERMARK_TTL_MS } from "../src/internal/redis-scripts.js"; -import { encodeFrame, FakeRedis } from "./fake-redis.js"; +import { decodeFrame, encodeFrame, FakeRedis } from "./fake-redis.js"; class RecordingMetrics implements DialCacheMetricsAdapter { readonly events: Array<{ readonly name: string; readonly labels: Record }> = []; @@ -122,7 +122,7 @@ describe("DialCache targeted invalidation watermarks", () => { expect(redis.readWatermarkValue(watermarkKey)).toBe(Date.parse("2026-05-12T18:00:00.000Z")); }); - it("stores a complete remote frame but does not publish local cache during a future invalidation window", async () => { + it("skips remote refills and local publication during an observed future invalidation window", async () => { const redis = new FakeRedis(); const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); let calls = 0; @@ -141,17 +141,79 @@ describe("DialCache targeted invalidation watermarks", () => { expect(first).toEqual({ userId: "123", calls: 1 }); expect(second).toEqual({ userId: "123", calls: 2 }); - expect([...redis.values.keys()].sort()).toEqual([ - watermarkKey, - valueKey("FutureBufferUser"), - ].sort()); + expect([...redis.values.keys()]).toEqual([watermarkKey]); + expect(redis.setCalls).toBe(1); await expect(redis.read({ valueKey: valueKey("FutureBufferUser"), watermarkKey, - })).resolves.toBeNull(); - await expect(redis.read({ valueKey: valueKey("FutureBufferUser") })).resolves.toMatchObject({ - payload: JSON.stringify(second), + })).resolves.toEqual({ + observedWatermarkMs: Date.parse("2026-05-12T18:00:01.000Z"), + }); + await expect(redis.read({ valueKey: valueKey("FutureBufferUser") })).resolves.toBeNull(); + }); + + it("writes the exact refill candidate when it is newer than the observed watermark", async () => { + const now = Date.now(); + const redis = new FakeRedis(); + redis.setRaw(valueKey("NewerRefillCandidate"), encodeFrame({ source: "old" }, now - 15)); + redis.setRaw(watermarkKey, String(now - 5)); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); + let calls = 0; + const getUser = dialcache.cached(async (userId: string) => ({ userId, calls: ++calls }), { + keyType: "user_id", + useCase: "NewerRefillCandidate", + cacheKey: (userId) => userId, + trackForInvalidation: true, + defaultConfig: remoteOnly(), + }); + + const first = await dialcache.enable(async () => await getUser("123")); + const second = await dialcache.enable(async () => await getUser("123")); + + expect(first).toEqual({ userId: "123", calls: 1 }); + expect(second).toEqual(first); + expect(calls).toBe(1); + expect(redis.mGetCalls).toBe(2); + expect(redis.setCalls).toBe(1); + expect(decodeFrame(redis.raw(valueKey("NewerRefillCandidate"))).createdAtMs).toBe(now); + }); + + it.each([ + { boundary: "equal to", watermarkOffsetMs: 0 }, + { boundary: "behind", watermarkOffsetMs: 10 }, + ])("skips a refill $boundary the observed watermark before serialization", async ({ watermarkOffsetMs }) => { + const now = Date.now(); + const useCase = `SkippedRefillCandidate${watermarkOffsetMs}`; + const redis = new FakeRedis(); + redis.setRaw(valueKey(useCase), encodeFrame({ source: "old" }, now - 20)); + redis.setRaw(watermarkKey, String(now + watermarkOffsetMs)); + const dump = vi.fn((): string => { + throw new Error("fenced refill must not serialize"); }); + const serializer: Serializer<{ userId: string; calls: number }> = { + dump, + load: () => { + throw new Error("fenced stale frame must not deserialize"); + }, + }; + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); + let calls = 0; + const getUser = dialcache.cached(async (userId: string) => ({ userId, calls: ++calls }), { + keyType: "user_id", + useCase, + cacheKey: (userId) => userId, + trackForInvalidation: true, + defaultConfig: remoteOnly(), + serializer, + }); + + const value = await dialcache.enable(async () => await getUser("123")); + + expect(value).toEqual({ userId: "123", calls: 1 }); + expect(dump).not.toHaveBeenCalled(); + expect(redis.mGetCalls).toBe(1); + expect(redis.setCalls).toBe(0); + expect(decodeFrame(redis.raw(valueKey(useCase))).createdAtMs).toBe(now - 20); }); it("stores but fences a write when invalidation arrives during fallback", async () => { @@ -185,7 +247,9 @@ describe("DialCache targeted invalidation watermarks", () => { await expect(redis.read({ valueKey: valueKey("FutureBufferFallbackRace"), watermarkKey, - })).resolves.toBeNull(); + })).resolves.toEqual({ + observedWatermarkMs: Date.parse("2026-05-12T18:00:01.000Z"), + }); }); it("stores but fences a write when invalidation remains active after slow serialization", async () => { @@ -237,7 +301,9 @@ describe("DialCache targeted invalidation watermarks", () => { await expect(redis.read({ valueKey: valueKey("FutureBufferSerializationRace"), watermarkKey, - })).resolves.toBeNull(); + })).resolves.toEqual({ + observedWatermarkMs: Date.parse("2026-05-12T18:00:01.000Z"), + }); }); it("fences a same-millisecond complete write for a zero-length future buffer", async () => { @@ -254,6 +320,8 @@ describe("DialCache targeted invalidation watermarks", () => { await dialcache.invalidateRemote("user_id", "123", 0); const fenced = await dialcache.enable(async () => await getUser("123")); + expect(redis.values.has(valueKey("ZeroBufferBoundary"))).toBe(false); + expect(redis.setCalls).toBe(1); vi.advanceTimersByTime(1); const written = await dialcache.enable(async () => await getUser("123")); const cached = await dialcache.enable(async () => await getUser("123")); @@ -262,6 +330,7 @@ describe("DialCache targeted invalidation watermarks", () => { expect(written).toEqual({ userId: "123", calls: 2 }); expect(cached).toEqual(written); expect(calls).toBe(2); + expect(redis.setCalls).toBe(2); }); it("serves tracked writes after the future buffer", async () => { diff --git a/test/dialcache-redis.test.ts b/test/dialcache-redis.test.ts index 951e8bd..15d9e8d 100644 --- a/test/dialcache-redis.test.ts +++ b/test/dialcache-redis.test.ts @@ -269,6 +269,45 @@ describe("DialCache Redis TTL layer", () => { expect(metrics.miss).toHaveBeenCalledOnce(); }); + it("preserves structurally compatible custom frames with extra watermark metadata", async () => { + const nowMs = 1_700_000_000_000; + vi.spyOn(Date, "now").mockReturnValue(nowMs); + const cachedValue = { source: "redis" }; + const redis: DialCacheRedisClient = { + read: vi.fn(async () => ({ + payload: JSON.stringify(cachedValue), + createdAtMs: nowMs, + observedWatermarkMs: nowMs - 1, + })), + write: vi.fn(async () => undefined), + invalidate: vi.fn(async () => undefined), + }; + const serializer: Serializer = { + dump: vi.fn((value) => JSON.stringify(value)), + load: vi.fn((value) => JSON.parse(Buffer.isBuffer(value) ? value.toString("utf8") : value)), + }; + const fallback = vi.fn(async () => ({ source: "fallback" })); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); + const getUser = dialcache.cached(fallback, { + keyType: "user_id", + useCase: "RedisAugmentedTrackedFrame", + cacheKey: () => "123", + trackForInvalidation: true, + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + ramp: { [CacheLayer.REMOTE]: 100 }, + }), + serializer, + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual(cachedValue); + + expect(serializer.load).toHaveBeenCalledOnce(); + expect(serializer.dump).not.toHaveBeenCalled(); + expect(fallback).not.toHaveBeenCalled(); + expect(redis.write).not.toHaveBeenCalled(); + }); + it.each([ Number.NaN, Number.POSITIVE_INFINITY, diff --git a/test/dialcache-shadow-confirmation.test.ts b/test/dialcache-shadow-confirmation.test.ts index 68f3665..134d92d 100644 --- a/test/dialcache-shadow-confirmation.test.ts +++ b/test/dialcache-shadow-confirmation.test.ts @@ -10,7 +10,6 @@ import { FallbackTimeoutError, type CacheMetricLabels, type CoalescedMetricLabels, - type DecodedRedisFrame, type DialCacheConfig, type DialCacheMetricsAdapter, type DialCacheRedisClient, @@ -20,8 +19,10 @@ import { type RedisCachePayload, type RedisInvalidationRequest, type RedisReadContext, + type RedisReadResult, type RedisReadRequest, type RedisWriteRequest, + type RedisWatermarkMiss, type SerializationMetricLabels, type Serializer, type ShadowValidationMetricLabels, @@ -53,7 +54,8 @@ function deferred(): Deferred { return { promise, resolve, reject }; } -type ReadStep = () => RedisCachePayload | null | Promise; +type ScriptedReadResult = RedisCachePayload | RedisWatermarkMiss | null; +type ReadStep = () => ScriptedReadResult | Promise; const MAX_TRACKED_REDIS_VALUE_TTL_MS = 60 * 60 * 1_000; @@ -66,21 +68,28 @@ class ScriptedRedis implements DialCacheRedisClient { constructor(private readonly steps: ReadStep[]) {} - async read(request: RedisReadRequest, context?: RedisReadContext): Promise { + async read(request: RedisReadRequest, context?: RedisReadContext): Promise { this.requests.push(request); this.contexts.push(context); const step = this.steps.shift(); if (step === undefined) { throw new Error("Unexpected Redis read"); } - const payload = await step(); - if (payload === null) { - return null; + const result = await step(); + if (result === null || isWatermarkMiss(result)) { + return result; } - return { payload, createdAtMs: this.frameCreatedAtMs }; + return { payload: result, createdAtMs: this.frameCreatedAtMs }; } } +function isWatermarkMiss(result: ScriptedReadResult): result is RedisWatermarkMiss { + return typeof result === "object" + && !Buffer.isBuffer(result) + && result !== null + && "observedWatermarkMs" in result; +} + type OrdinaryMetricName = | "request" | "miss" @@ -602,6 +611,36 @@ describe("DialCache Redis shadow confirmation", () => { expectTrackedReads(redis, 2); }); + it("reports superseded when C1 becomes a typed watermark miss", async () => { + const payload = JSON.stringify({ id: "123", version: 1 }); + const frameCreatedAtMs = Date.now(); + const redis = new ScriptedRedis([ + () => payload, + () => ({ observedWatermarkMs: frameCreatedAtMs + 1 }), + ]); + redis.frameCreatedAtMs = frameCreatedAtMs; + const metrics = new RecordingMetrics(); + const serializer: Serializer<{ readonly id: string; readonly version: number }> = { + dump: vi.fn((value) => JSON.stringify(value)), + load: vi.fn((value) => JSON.parse(Buffer.isBuffer(value) ? value.toString("utf8") : value)), + }; + const dialcache = createCache(redis, metrics); + const getUser = dialcache.cached(async () => ({ id: "123", version: 2 }), { + ...trackedOptions("ShadowSupersededWatermarkMiss", remoteConfig(100)), + cacheKey: () => "123", + serializer, + }); + + await dialcache.enable(async () => await getUser()); + await waitForShadowEvents(metrics, 1); + + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["superseded"]); + expect(metrics.shadowAgeEvents).toEqual([]); + expect(serializer.load).toHaveBeenCalledTimes(2); + expect(serializer.dump).not.toHaveBeenCalled(); + expectTrackedReads(redis, 2); + }); + it("retains a future-dated C1 for payload confirmation and attributes its offset to remote_shadow", async () => { const nowMs = 1_700_000_000_000; const nowSpy = vi.spyOn(Date, "now").mockReturnValue(nowMs); @@ -1335,6 +1374,7 @@ describe("DialCache Redis shadow confirmation", () => { value: JSON.stringify({ id: "123" }), })); expect(Object.hasOwn(redis.write.mock.calls[0]?.[0] ?? {}, "watermarkKey")).toBe(false); + expect(Object.hasOwn(redis.write.mock.calls[0]?.[0] ?? {}, "createdAtMs")).toBe(false); expect(metrics.ordinaryEvents.filter(({ name, labels }) => name === "request" && labels.layer === REMOTE_SHADOW_CACHE_LAYER )).toHaveLength(1); @@ -1354,6 +1394,52 @@ describe("DialCache Redis shadow confirmation", () => { )).toHaveLength(1); }); + it("reports fill_fenced and skips shadow serialization when the candidate cannot clear the observed watermark", async () => { + const nowMs = 1_700_000_000_000; + const nowSpy = vi.spyOn(Date, "now").mockReturnValue(nowMs); + try { + const redis = new ScriptedRedis([() => ({ observedWatermarkMs: nowMs })]); + const metrics = new RecordingMetrics(); + const serializer: Serializer<{ readonly id: string }> = { + dump: vi.fn(() => { + throw new Error("fenced shadow fill must not serialize"); + }), + load: vi.fn(() => { + throw new Error("watermark miss must not deserialize"); + }), + }; + const dialcache = createCache(redis, metrics); + const getUser = dialcache.cached(async () => ({ id: "123" }), { + ...trackedOptions("ShadowDarkFillFenced", remoteConfig(0)), + cacheKey: () => "123", + serializer, + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual({ id: "123" }); + await waitForShadowEvents(metrics, 1); + + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["fill_fenced"]); + expect(serializer.dump).not.toHaveBeenCalled(); + expect(serializer.load).not.toHaveBeenCalled(); + expect(redis.write).not.toHaveBeenCalled(); + expectTrackedReads(redis, 1); + expect(metrics.ordinaryEvents.filter(({ name, labels }) => + name === "request" && labels.layer === REMOTE_SHADOW_CACHE_LAYER + )).toHaveLength(1); + expect(metrics.ordinaryEvents.filter(({ name, labels }) => + name === "miss" && labels.layer === REMOTE_SHADOW_CACHE_LAYER + )).toHaveLength(1); + expect(metrics.ordinaryEvents.filter(({ name, labels }) => + name === "get" && labels.layer === REMOTE_SHADOW_CACHE_LAYER + )).toHaveLength(1); + expect(metrics.ordinaryEvents.filter(({ name, labels }) => + (name === "serialization" || name === "size") && labels.layer === REMOTE_SHADOW_CACHE_LAYER + )).toHaveLength(0); + } finally { + nowSpy.mockRestore(); + } + }); + it("reports a detached serializer dump failure as fill_error with an exact remote_shadow error", async () => { const redis = new ScriptedRedis([() => null]); const metrics = new RecordingMetrics(); diff --git a/test/dialcache-stale-on-error.test.ts b/test/dialcache-stale-on-error.test.ts index dc8b633..3e11045 100644 --- a/test/dialcache-stale-on-error.test.ts +++ b/test/dialcache-stale-on-error.test.ts @@ -8,8 +8,8 @@ import { DialCacheKey, DialCacheKeyConfig, type CachedOptions, - type DecodedRedisFrame, type DialCacheMetricsAdapter, + type RedisReadResult, type RedisReadContext, type RedisReadRequest, type Serializer, @@ -30,7 +30,7 @@ class RecordingRedis extends FakeRedis { override async read( request: RedisReadRequest, context?: RedisReadContext, - ): Promise { + ): Promise { this.readRequests.push(request); this.readContexts.push(context); return await super.read(request); @@ -48,11 +48,11 @@ class HangingReadRedis extends FakeRedis { override async read( request: RedisReadRequest, context?: RedisReadContext, - ): Promise { + ): Promise { this.readRequests.push(request); this.readContexts.push(context); if (this.readRequests.length === this.hangOnCall) { - return await new Promise(() => undefined); + return await new Promise(() => undefined); } return await super.read(request); } diff --git a/test/fake-redis.ts b/test/fake-redis.ts index 2a004f2..e443b32 100644 --- a/test/fake-redis.ts +++ b/test/fake-redis.ts @@ -1,9 +1,10 @@ import type { - DecodedRedisFrame, DialCacheRedisClient, RedisInvalidationRequest, + RedisReadResult, RedisReadRequest, RedisWriteRequest, + RedisWatermarkMiss, } from "../src/index.js"; import { MAX_TRACKED_REDIS_VALUE_TTL_MS } from "../src/internal/duration.js"; import { MIN_WATERMARK_TTL_MS } from "../src/internal/redis-scripts.js"; @@ -30,7 +31,7 @@ export class FakeRedis implements DialCacheRedisClient { failWatermarkGet = false; getGate: Promise | null = null; - async read({ valueKey, watermarkKey }: RedisReadRequest): Promise { + async read({ valueKey, watermarkKey }: RedisReadRequest): Promise { if (watermarkKey === undefined) { this.getCalls += 1; } else { @@ -45,15 +46,16 @@ export class FakeRedis implements DialCacheRedisClient { valueKey, cacheTtlMs, value, + createdAtMs, }: RedisWriteRequest): Promise { const validatedTtlMs = ceilSupportedCacheTtlMs(cacheTtlMs); - const createdAtMs = Date.now(); - const frame = encodeRedisFrame(value, createdAtMs); + const storedAtMs = Date.now(); + const frame = encodeRedisFrame(value, createdAtMs === undefined ? storedAtMs : createdAtMs); this.setCalls += 1; this.throwIfWriteFails(); this.values.set(valueKey, { value: frame, - expiresAtMs: createdAtMs + validatedTtlMs, + expiresAtMs: storedAtMs + validatedTtlMs, }); } @@ -118,25 +120,30 @@ export class FakeRedis implements DialCacheRedisClient { } } - private readPayload(valueKey: string, watermarkKey: string | null): DecodedRedisFrame | null { - const raw = this.readRaw(valueKey); - if (raw === null || raw.length < PAYLOAD_OFFSET || raw[0] !== FRAME_VERSION) { - return null; - } - - const createdAtMs = Number(readTimestamp(raw)); + private readPayload(valueKey: string, watermarkKey: string | null): RedisReadResult { + let watermark: number | null = null; + let watermarkMiss: RedisWatermarkMiss | null = null; if (watermarkKey !== null) { - let watermark: number | null; try { watermark = this.readWatermark(watermarkKey); } catch { return null; } - if (createdAtMs <= (watermark ?? 0)) { - return null; + if (watermark !== null) { + watermarkMiss = { observedWatermarkMs: watermark }; } } + const raw = this.readRaw(valueKey); + if (raw === null || raw.length < PAYLOAD_OFFSET || raw[0] !== FRAME_VERSION) { + return watermarkMiss; + } + + const createdAtMs = Number(readTimestamp(raw)); + if (createdAtMs <= (watermark ?? 0)) { + return watermarkMiss; + } + const encoding = raw[ENCODING_OFFSET]; if (encoding === 0) { return { payload: raw.subarray(PAYLOAD_OFFSET).toString("utf8"), createdAtMs }; diff --git a/test/node-redis.test.ts b/test/node-redis.test.ts index 3867fee..edf639a 100644 --- a/test/node-redis.test.ts +++ b/test/node-redis.test.ts @@ -128,6 +128,18 @@ describe("node-redis adapter", () => { ).resolves.toBeUndefined(); }); + it("returns an observed watermark for tracked semantic misses", async () => { + const client = fakeClient({ mGet: [null, Buffer.from("1234")] }); + const adapter = createNodeRedisDialCacheClient(client as never); + + await expect(adapter.read({ + valueKey: "tracked:{id}:value", + watermarkKey: "tracked:{id}:watermark", + })).resolves.toEqual({ observedWatermarkMs: 1_234 }); + + expect(client.sendCommand).toHaveBeenCalledTimes(1); + }); + it("writes complete frames with one native SET", async () => { vi.spyOn(Date, "now").mockReturnValue(1_234); const client = fakeClient(); @@ -150,6 +162,25 @@ describe("node-redis adapter", () => { expect(options).toMatchObject({ returnBuffers: true }); }); + it("honors a supplied write timestamp without sampling the client clock", async () => { + const now = vi.spyOn(Date, "now").mockImplementation(() => { + throw new Error("Date.now must not be sampled for a supplied timestamp"); + }); + const client = fakeClient(); + const adapter = createNodeRedisDialCacheClient(client as never); + + await expect(adapter.write({ + valueKey: "tracked:{id}:value", + cacheTtlMs: 1_000, + value: "tracked", + createdAtMs: 0, + })).resolves.toBeUndefined(); + + const [args] = client.sendCommand.mock.calls[0] as [Array]; + expect(Number((args[2] as Buffer).readBigUInt64BE(1))).toBe(0); + expect(now).not.toHaveBeenCalled(); + }); + it("writes binary values as complete frames with one client-clock sample", async () => { const now = vi.spyOn(Date, "now").mockReturnValue(1_234); const client = fakeClient(); diff --git a/test/prometheus.test.ts b/test/prometheus.test.ts index 21e0e21..ceff5d6 100644 --- a/test/prometheus.test.ts +++ b/test/prometheus.test.ts @@ -74,6 +74,7 @@ const SHADOW_VALIDATION_OUTCOMES: Readonly mismatch: true, superseded: true, filled: true, + fill_fenced: true, fill_error: true, redis_error: true, source_error: true, diff --git a/test/redis-cluster.integration.test.ts b/test/redis-cluster.integration.test.ts index 40b3c43..dea2967 100644 --- a/test/redis-cluster.integration.test.ts +++ b/test/redis-cluster.integration.test.ts @@ -267,6 +267,9 @@ describe("DialCache Redis protocol on Redis Cluster", () => { if (glideCluster === undefined) { return ctx.skip(); } + if (cluster === undefined) { + throw new Error("Redis Cluster did not start"); + } const adapter = createValkeyGlideDialCacheClient(glideCluster, valkeyGlide); const valueKey = "glide-cluster:{item:tracked}:value"; const watermarkKey = "glide-cluster:{item:tracked}:watermark"; @@ -289,7 +292,11 @@ describe("DialCache Redis protocol on Redis Cluster", () => { // The existing value remains fenced until a later client-stamped frame is // written past the zero-buffer watermark. await new Promise((resolve) => setTimeout(resolve, 25)); - expect(await adapter.read({ valueKey, watermarkKey })).toBeNull(); + const observedWatermark = await cluster.get(watermarkKey); + expect(observedWatermark).not.toBeNull(); + expect(await adapter.read({ valueKey, watermarkKey })).toEqual({ + observedWatermarkMs: Number(observedWatermark), + }); await expect( adapter.write({ valueKey, cacheTtlMs: 60_000, value: "glide-2" }), ).resolves.toBeUndefined(); @@ -328,6 +335,10 @@ describe("DialCache Redis protocol on Redis Cluster", () => { await flushAllMasters(); await expect(adapter.invalidate({ watermarkKey, futureBufferMs: 0 })).resolves.toBeUndefined(); - expect(await adapter.read({ valueKey, watermarkKey })).toBeNull(); + const observedWatermark = await cluster.get(watermarkKey); + expect(observedWatermark).not.toBeNull(); + expect(await adapter.read({ valueKey, watermarkKey })).toEqual({ + observedWatermarkMs: Number(observedWatermark), + }); }); }); diff --git a/test/redis-payload.test.ts b/test/redis-payload.test.ts index 0da4010..82b0b2e 100644 --- a/test/redis-payload.test.ts +++ b/test/redis-payload.test.ts @@ -1,6 +1,7 @@ import { decodeRedisFrame, decodeTrackedRedisFrame, + decodeTrackedRedisReadResult, encodeRedisFrame, } from "../src/redis-protocol.js"; import { @@ -60,6 +61,12 @@ describe("Redis frame decoding", () => { expect(() => decodeRedisFrame(reply)).toThrow(DialCacheRedisPayloadError); expect(() => decodeTrackedRedisFrame(reply, null)).toThrow(DialCacheRedisPayloadError); expect(() => decodeTrackedRedisFrame(null, reply)).toThrow(DialCacheRedisPayloadError); + expect(() => decodeTrackedRedisReadResult(reply, null)).toThrow( + DialCacheRedisPayloadError, + ); + expect(() => decodeTrackedRedisReadResult(null, reply)).toThrow( + DialCacheRedisPayloadError, + ); } }); @@ -79,6 +86,42 @@ describe("Redis frame decoding", () => { ).toBeNull(); }); + it("preserves valid observed watermarks for every tracked semantic miss", () => { + const watermark = Buffer.from("1000"); + + for (const frame of [ + null, + Buffer.alloc(9), + encodeFrame("unsupported", 0, 2_000, 2), + encodeFrame("fenced", 0, 1_000), + ]) { + expect(decodeTrackedRedisReadResult(frame, watermark)).toEqual({ + observedWatermarkMs: 1_000, + }); + expect(decodeTrackedRedisFrame(frame, watermark)).toBeNull(); + } + }); + + it("retains generic misses when no trustworthy watermark was observed", () => { + for (const watermark of [null, Buffer.from("invalid")]) { + expect(decodeTrackedRedisReadResult(null, watermark)).toBeNull(); + expect(decodeTrackedRedisReadResult(Buffer.alloc(9), watermark)).toBeNull(); + expect(decodeTrackedRedisReadResult( + encodeFrame("unsupported", 0, 2_000, 2), + watermark, + )).toBeNull(); + } + }); + + it("returns eligible tracked frames through the typed decoder", () => { + expect( + decodeTrackedRedisReadResult( + encodeFrame("cached", 0, 1_001), + Buffer.from("1000"), + ), + ).toEqual({ payload: "cached", createdAtMs: 1_001 }); + }); + it("treats a missing watermark as the zero baseline", () => { const frame = encodeFrame("cached", 0, 1_000); diff --git a/test/redis-real.integration.test.ts b/test/redis-real.integration.test.ts index 7d13152..c221908 100644 --- a/test/redis-real.integration.test.ts +++ b/test/redis-real.integration.test.ts @@ -445,6 +445,59 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(redisRead).toHaveBeenCalledOnce(); }); + it("conditionally skips and then admits a tracked refill from the observed watermark", async () => { + if (client === undefined || admin === undefined) { + throw new Error("Redis test clients did not start"); + } + const namespace = "real-conditional-refill"; + const useCase = "RealConditionalRefill"; + const valueKey = `{${namespace}:item_id:refill}#${useCase}:dialcache-frame-v1`; + const watermarkKey = `{${namespace}:item_id:refill}#watermark`; + const candidateAtMs = 1_700_000_000_100; + const staleFrame = encodeFrame(JSON.stringify({ id: "refill", source: "stale" }), 0, candidateAtMs - 10); + await admin.set(valueKey, staleFrame, { PX: 60_000 }); + await admin.set(watermarkKey, String(candidateAtMs), { PX: 60_000 }); + + const write = vi.fn(client.adapter.write); + const redisClient: DialCacheRedisClient = { ...client.adapter, write }; + const dialcache = new DialCache({ + namespace, + redis: { client: redisClient, readTimeoutMs: 10_000 }, + }); + let calls = 0; + const getPayload = dialcache.cached(async () => ({ id: "refill", calls: ++calls }), { + keyType: "item_id", + useCase, + cacheKey: () => "refill", + trackForInvalidation: true, + defaultConfig: remoteOnly, + }); + const now = vi.spyOn(Date, "now").mockReturnValue(candidateAtMs); + try { + const fenced = await dialcache.enable(async () => await getPayload()); + expect(fenced).toEqual({ id: "refill", calls: 1 }); + expect(write).not.toHaveBeenCalled(); + expect(await admin.get(commandOptions({ returnBuffers: true }), valueKey)).toEqual(staleFrame); + + now.mockReturnValue(candidateAtMs + 1); + const written = await dialcache.enable(async () => await getPayload()); + const cached = await dialcache.enable(async () => await getPayload()); + + expect(written).toEqual({ id: "refill", calls: 2 }); + expect(cached).toEqual(written); + expect(calls).toBe(2); + expect(write).toHaveBeenCalledOnce(); + expect(write).toHaveBeenCalledWith(expect.objectContaining({ + valueKey, + createdAtMs: candidateAtMs + 1, + })); + const stored = await admin.get(commandOptions({ returnBuffers: true }), valueKey); + expect(stored?.readBigUInt64BE(1)).toBe(BigInt(candidateAtMs + 1)); + } finally { + now.mockRestore(); + } + }); + it("compresses values above the threshold and stores small values byte-identical", async () => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); @@ -1062,7 +1115,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(invalidate).not.toHaveBeenCalled(); }); - it("stores but fences a shadow fill behind a future watermark", async () => { + it("skips a shadow fill behind a future watermark", async () => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); } @@ -1070,14 +1123,13 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { const useCase = "RealDarkShadowFenced"; const valueKey = `{${namespace}:item_id:fenced}#${useCase}:dialcache-frame-v1`; const watermarkKey = `{${namespace}:item_id:fenced}#watermark`; - await client.adapter.invalidate({ watermarkKey, futureBufferMs: 60_000 }); - const watermarkBefore = await admin.get(watermarkKey); + const candidateAtMs = 1_700_000_000_100; const read = vi.fn(client.adapter.read); const write = vi.fn(client.adapter.write); const invalidate = vi.fn(client.adapter.invalidate); const redisClient: DialCacheRedisClient = { ...client.adapter, read, write, invalidate }; - const filled = deferred(); + const fenced = deferred(); const metrics: DialCacheMetricsAdapter = { request: vi.fn(), miss: vi.fn(), @@ -1086,8 +1138,8 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { invalidation: vi.fn(), coalesced: vi.fn(), shadowValidation: vi.fn(({ outcome }) => { - if (outcome === "filled") { - filled.resolve(); + if (outcome === "fill_fenced") { + fenced.resolve(); } }), observeGet: vi.fn(), @@ -1114,23 +1166,33 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { }), }); - const result = await dialcache.enable(async () => await getPayload()); - expect(result).toBe(sourceValue); - await filled.promise; + const now = vi.spyOn(Date, "now").mockReturnValue(candidateAtMs); + try { + await client.adapter.invalidate({ watermarkKey, futureBufferMs: 0 }); + expect(await admin.get(watermarkKey)).toBe(String(candidateAtMs)); + + const result = await dialcache.enable(async () => await getPayload()); + expect(result).toBe(sourceValue); + await fenced.promise; + } finally { + now.mockRestore(); + } expect(source).toHaveBeenCalledOnce(); expect(read).toHaveBeenCalledOnce(); - expect(write).toHaveBeenCalledOnce(); + expect(write).not.toHaveBeenCalled(); expect(invalidate).not.toHaveBeenCalled(); - expect(await admin.exists(valueKey)).toBe(1); - expect(await client.adapter.read({ valueKey, watermarkKey })).toBeNull(); - expect(await admin.get(watermarkKey)).toBe(watermarkBefore); + expect(await admin.exists(valueKey)).toBe(0); + expect(await client.adapter.read({ valueKey, watermarkKey })).toEqual({ + observedWatermarkMs: candidateAtMs, + }); + expect(await admin.get(watermarkKey)).toBe(String(candidateAtMs)); expect(metrics.shadowValidation).toHaveBeenCalledOnce(); expect(metrics.shadowValidation).toHaveBeenCalledWith({ cacheNamespace: namespace, useCase, keyType: "item_id", - outcome: "filled", + outcome: "fill_fenced", }); }); @@ -1166,7 +1228,10 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { }), ).resolves.toBeUndefined(); expect(await admin.scriptExists(INVALIDATE_CACHE_SHA1)).toEqual([true]); - expect(await scriptClient.read({ valueKey: trackedValueKey, watermarkKey })).toBeNull(); + const watermark = Number(await admin.get(watermarkKey)); + expect(await scriptClient.read({ valueKey: trackedValueKey, watermarkKey })).toEqual({ + observedWatermarkMs: watermark, + }); }); it("uses zero for a missing watermark and misses on malformed or fenced state", async () => { @@ -1195,7 +1260,9 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); await admin.set(watermarkKey, "1000"); - expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); + expect(await scriptClient.read({ valueKey, watermarkKey })).toEqual({ + observedWatermarkMs: 1_000, + }); await admin.set(watermarkKey, "999.5"); expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); @@ -1274,7 +1341,9 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { await admin.hSet(valueKey, "field", "value"); await admin.set(watermarkKey, "0"); await expect(scriptClient.read({ valueKey })).rejects.toThrow(/WRONGTYPE/); - await expect(scriptClient.read({ valueKey, watermarkKey })).resolves.toBeNull(); + await expect(scriptClient.read({ valueKey, watermarkKey })).resolves.toEqual({ + observedWatermarkMs: 0, + }); await admin.del([valueKey, watermarkKey]); await admin.set(valueKey, encodeFrame("cached", 0, 1_000)); @@ -1297,7 +1366,9 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(await admin.get(watermarkKey)).toBe(String(invalidatedAtMs + 100)); expect(await admin.pTTL(watermarkKey)).toBeGreaterThan(MIN_WATERMARK_TTL_MS - 1_000); expect(await admin.pTTL(watermarkKey)).toBeLessThanOrEqual(MIN_WATERMARK_TTL_MS); - expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); + expect(await scriptClient.read({ valueKey, watermarkKey })).toEqual({ + observedWatermarkMs: invalidatedAtMs + 100, + }); const namespace = "wrong-type-repair"; const repairValueKey = `{${namespace}:item_id:repair}#WrongTypeRepair:dialcache-frame-v1`; @@ -1738,12 +1809,16 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect((await scriptClient.read({ valueKey, watermarkKey }))?.payload).toBe("cached"); await scriptClient.invalidate({ watermarkKey, futureBufferMs: 100 }); - expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); + expect(await scriptClient.read({ valueKey, watermarkKey })).toEqual({ + observedWatermarkMs: invalidatedAtMs + 100, + }); const watermarkBeforeWrite = await admin.get(watermarkKey); const watermarkTtlBeforeWrite = await admin.pTTL(watermarkKey); await scriptClient.write({ ...writeRequest, value: "behind-watermark" }); expect((await scriptClient.read({ valueKey }))?.payload).toBe("behind-watermark"); - expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); + expect(await scriptClient.read({ valueKey, watermarkKey })).toEqual({ + observedWatermarkMs: invalidatedAtMs + 100, + }); expect(await admin.get(watermarkKey)).toBe(watermarkBeforeWrite); const watermarkTtlAfterWrite = await admin.pTTL(watermarkKey); expect(watermarkTtlAfterWrite).toBeGreaterThan(watermarkTtlBeforeWrite - 1_000); @@ -1770,7 +1845,10 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { await scriptClient.write({ valueKey, cacheTtlMs: 60_000, value: "stale" }); await scriptClient.invalidate({ watermarkKey, futureBufferMs: 60_000 }); - expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); + const watermark = Number(await admin.get(watermarkKey)); + expect(await scriptClient.read({ valueKey, watermarkKey })).toEqual({ + observedWatermarkMs: watermark, + }); await admin.del(watermarkKey); diff --git a/test/valkey-glide.test.ts b/test/valkey-glide.test.ts index 8fed18f..56f7a2d 100644 --- a/test/valkey-glide.test.ts +++ b/test/valkey-glide.test.ts @@ -175,6 +175,18 @@ describe("Valkey GLIDE adapter", () => { expect(client.customCommand).not.toHaveBeenCalled(); }); + it("returns an observed watermark for tracked semantic misses", async () => { + const client = fakeClient([[null, Buffer.from("1234")]]); + const adapter = createValkeyGlideDialCacheClient(client, mockGlide); + + await expect(adapter.read({ + valueKey: "tracked:{id}:value", + watermarkKey: "tracked:{id}:watermark", + })).resolves.toEqual({ observedWatermarkMs: 1_234 }); + + expect(client.exec).toHaveBeenCalledTimes(1); + }); + it("routes tracked cluster MGET directly to the slot primary", async () => { const client = fakeClusterClient([ redisFrame("tracked-cluster"), @@ -340,6 +352,25 @@ describe("Valkey GLIDE adapter", () => { expect(now).toHaveBeenCalledTimes(3); }); + it("honors a supplied write timestamp without sampling the client clock", async () => { + const now = vi.spyOn(Date, "now").mockImplementation(() => { + throw new Error("Date.now must not be sampled for a supplied timestamp"); + }); + const client = fakeClient("OK"); + const adapter = createValkeyGlideDialCacheClient(client, mockGlide); + + await expect(adapter.write({ + valueKey: "tracked:{id}:value", + cacheTtlMs: 1_000, + value: "tracked", + createdAtMs: 0, + })).resolves.toBeUndefined(); + + const [args] = client.customCommand.mock.calls[0] ?? [[]]; + expect(Number((args[2] as Buffer).readBigUInt64BE(1))).toBe(0); + expect(now).not.toHaveBeenCalled(); + }); + it("routes cluster writes and invalidations to the slot primary", async () => { const client = fakeClusterClient("OK", "OK", 1); const adapter = createValkeyGlideDialCacheClient(client, mockGlide); From f63afad56c365b801cef820df13b29f67f649249 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sun, 30 Aug 2026 10:23:25 -0700 Subject: [PATCH 2/2] fix(redis): preserve admitted refill freshness --- README.md | 22 +++--- scripts/test-package.mjs | 15 ++++- src/dialcache.ts | 19 ++---- src/internal/redis-cache.ts | 59 ++++++++-------- src/internal/redis-payload.ts | 13 ++-- src/redis-client.ts | 19 +++++- test/dialcache-invalidation.test.ts | 78 ++++++++++++++++++++++ test/dialcache-redis.test.ts | 57 +++++++++++++++- test/dialcache-shadow-confirmation.test.ts | 48 ++++++++++++- test/fake-redis.ts | 2 +- test/node-redis.test.ts | 2 +- test/redis-cluster.integration.test.ts | 2 + test/redis-payload.test.ts | 1 + test/redis-real.integration.test.ts | 8 +++ test/valkey-glide.test.ts | 2 +- 15 files changed, 276 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index 6874983..837550a 100644 --- a/README.md +++ b/README.md @@ -408,9 +408,9 @@ Awaiting those public promises does not drain detached shadow work. Shadow sched Neither adapter owns additional resources or native script handles, so the application simply closes the underlying client after draining work. -Reads use native `GET` for untracked entries and one atomic `MGET` for each tracked value-and-watermark pair. The adapters validate and decode the returned frame in the Node process. Tracked reads are deliberately routed to primaries so a lagging replica cannot hide an invalidation watermark. When a tracked semantic miss observes a present, valid numeric watermark, the bundled adapters return a typed `RedisWatermarkMiss` carrying that watermark; generic and legacy misses remain `null`. After a read settles, DialCache evaluates the frame against the observing application's `Date.now()`. Without stale-on-error, the read accepts only nonnegative ages strictly below the effective remote TTL `F`. With a positive recovery maximum `M`, that same initial read is bounded by `M`: ages below `F` deserialize and serve normally, while ages from `F` through strictly below `M` remain raw as a possible source-error recovery candidate. Future-dated frames fail closed before deserialization and emit the bounded offset observation described under [Metrics](#metrics). A shadow confirmation read still observes a future offset but retains the payload only for supersession comparison; it can never serve that frame. +Reads use native `GET` for untracked entries and one atomic `MGET` for each tracked value-and-watermark pair. The adapters validate and decode the returned frame in the Node process. Tracked reads are deliberately routed to primaries so a lagging replica cannot hide an invalidation watermark. When a tracked semantic miss observes a present, valid numeric watermark, the bundled adapters return `RedisWatermarkMiss { kind: "watermark_miss", observedWatermarkMs }`; generic and legacy misses remain `null`. After a read settles, DialCache evaluates the frame against the observing application's `Date.now()`. Without stale-on-error, the read accepts only nonnegative ages strictly below the effective remote TTL `F`. With a positive recovery maximum `M`, that same initial read is bounded by `M`: ages below `F` deserialize and serve normally, while ages from `F` through strictly below `M` remain raw as a possible source-error recovery candidate. Future-dated frames fail closed before deserialization and emit the bounded offset observation described under [Metrics](#metrics). A shadow confirmation read still observes a future offset but retains the payload only for supersession comparison; it can never serve that frame. -Writes are native too, so the payload never crosses the Redis-to-Lua boundary. For a DialCache fill following a typed tracked miss, core samples one candidate `createdAtMs = Date.now()` after the fallback succeeds and before `serializer.dump`, compression, or frame construction. If that candidate is at or before the observed watermark, DialCache skips the known-fenced refill at that point: it does not invoke the serializer, compress or allocate the payload/frame, or dispatch `SET`. Otherwise core passes the same candidate through optional `RedisWriteRequest.createdAtMs`, and the bundled adapter encodes that exact value. Ordinary, untracked, missing/malformed-watermark, and legacy-adapter misses leave the optional timestamp absent, preserving the adapter-side `Date.now()` sample immediately before dispatch. Every dispatched write issues one `SET valueKey frame PX cacheTtlMs` containing the complete version-1 frame; only tracked reads include the watermark key. With stale-on-error enabled, the requested physical TTL is `M` instead of `F`. `M` remains the configured logical recovery ceiling even when it exceeds one hour. Core separately caps every tracked Redis value's physical TTL at one hour, so such a tracked candidate may be evicted by expiry before it reaches logical age `M`; untracked Redis and local TTLs retain their configured limits. Each dispatched tracked write whose requested TTL exceeds that cap emits `error="tracked_ttl_clamped"`. The write never reads, creates, or extends a watermark. A dispatched frame can still be fenced if the watermark advances after the read snapshot. Same-key writes are ordinary Redis last-writer-wins operations, with no Lua, pipeline, or transaction on the write path. +Writes are native too, so the payload never crosses the Redis-to-Lua boundary. Conditional tracked refills follow the two-sample fence described under [targeted invalidation](#targeted-invalidation-and-watermarks): a preflight can avoid payload preparation, while an admitted fill uses a final dispatch-adjacent timestamp so serialization time does not consume its logical TTL. Ordinary, untracked, missing/malformed-watermark, and legacy-adapter misses leave optional `RedisWriteRequest.createdAtMs` absent, preserving the adapter-side `Date.now()` sample immediately before dispatch. Every dispatched write issues one `SET valueKey frame PX cacheTtlMs` containing the complete version-1 frame; only tracked reads include the watermark key. With stale-on-error enabled, the requested physical TTL is `M` instead of `F`. `M` remains the configured logical recovery ceiling even when it exceeds one hour. Core separately caps every tracked Redis value's physical TTL at one hour, so such a tracked candidate may be evicted by expiry before it reaches logical age `M`; untracked Redis and local TTLs retain their configured limits. Each dispatched tracked write whose requested TTL exceeds that cap emits `error="tracked_ttl_clamped"`. The write never reads, creates, or extends a watermark. A dispatched frame can still be fenced if the watermark advances after the read snapshot. Same-key writes are ordinary Redis last-writer-wins operations, with no Lua, pipeline, or transaction on the write path. The network shape remains one top-level Redis command and one round trip per semantic read (`GET` or `MGET`) and one `SET` per dispatched write. Stale recovery reuses the frame returned by that initial command and never adds a second Redis read, including after a source rejection. Retaining a raw candidate instead consumes process memory until that source attempt settles, once per distinct in-flight key (same-key coalesced callers share it). Conditional refill suppression reuses the existing tracked `MGET` result and adds no command or round trip. DialCache does not call Redis `TIME` or maintain a Redis-clock offset. Use the maintainer benchmarks below to measure the target Redis/Valkey version, payload distribution, and high-cardinality in-flight memory exposure. @@ -482,9 +482,9 @@ Writes, invalidations, async `cacheConfigProvider` calls, and custom serializer #### Serialization -The core Redis boundary is the client-agnostic `DialCacheRedisClient` interface. Writes accept serialized values as `string | Buffer`; reads return `RedisReadResult`, which is a `DecodedRedisFrame` — the decoded `string | Buffer` payload plus the frame header's writer-client `createdAtMs` — a typed `RedisWatermarkMiss`, or `null`. The interface does not expose client commands or wire encodings. A decoded frame's timestamp is correctness-relevant: custom clients must return the frame's valid nonnegative safe-integer epoch timestamp rather than a constant. `RedisWriteRequest` contains `valueKey`, `cacheTtlMs`, `value`, and optional `createdAtMs`, and `write()` returns `void`; watermark ownership remains exclusive to tracked reads and invalidation. Core supplies `createdAtMs` only when a typed miss makes that exact candidate part of the refill decision. A custom client that continues returning only `DecodedRedisFrame | null` remains correct and source-compatible, but does not enable the known-fenced refill optimization. A custom client that returns `RedisWatermarkMiss` opts into that optimization and must encode a supplied `RedisWriteRequest.createdAtMs` exactly so the decision timestamp and stored frame cannot diverge. +The core Redis boundary is the client-agnostic `DialCacheRedisClient` interface. Writes accept serialized values as `string | Buffer`; reads return `RedisReadResult`, which is a `DecodedRedisFrame` — the decoded `string | Buffer` payload plus the frame header's writer-client `createdAtMs` — `RedisWatermarkMiss { kind: "watermark_miss", observedWatermarkMs }`, or `null`. The interface does not expose client commands or wire encodings. A decoded frame's timestamp is correctness-relevant: custom clients must return the frame's valid nonnegative safe-integer epoch timestamp rather than a constant. `RedisWriteRequest` contains `valueKey`, `cacheTtlMs`, `value`, and optional `createdAtMs`, and `write()` returns `void`; watermark ownership remains exclusive to tracked reads and invalidation. Core supplies `createdAtMs` only after a discriminated miss passes both the preflight and final watermark checks; the supplied value is the final dispatch-adjacent sample. A custom client that continues returning only `DecodedRedisFrame | null` remains correct and source-compatible, but does not enable the known-fenced refill optimization. A custom client that returns the `RedisWatermarkMiss` variant opts into that optimization and must encode a supplied `RedisWriteRequest.createdAtMs` exactly so the final fence decision and stored frame cannot diverge. -The shared `encodeRedisFrame`, `decodeRedisFrame`, `decodeTrackedRedisFrame`, and `decodeTrackedRedisReadResult` helpers, the `validateRedisSetReply` and `validateRedisScriptInvalidationReply` reply helpers, the `ceilSupportedCacheTtlMs` TTL guard, and the invalidation Lua source are available from `dialcache/redis-protocol`. A custom write chooses `const createdAtMs = request.createdAtMs === undefined ? Date.now() : request.createdAtMs`, calls `encodeRedisFrame(request.value, createdAtMs)`, and sends one `SET valueKey frame PX cacheTtlMs` after validating the TTL. The fallback clock covers ordinary core writes and direct callers that omit the optional field; supplied values must not be resampled or replaced, and invalid runtime values must still be rejected by the frame encoder. A custom tracked read atomically obtains `[value, watermark]` from the primary. Passing both replies to `decodeTrackedRedisReadResult` opts into typed watermark misses; the backward-compatible `decodeTrackedRedisFrame` collapses those misses to `null` and preserves the established `DecodedRedisFrame | null` surface. A missing watermark is treated as zero for valid frames, while a malformed numeric watermark fails closed as a generic miss. Invalidation passes `KEYS = [watermarkKey]` and `ARGV = [futureBufferMs, invalidatedAtMs]`, with one stable client timestamp reused across retries. Custom adapters can throw the root-exported `DialCacheRedisPayloadError`, `DialCacheRedisPayloadEncodingError`, and `DialCacheRedisProtocolError` classes to distinguish malformed replies, unsupported encodings, and reply-domain violations. DialCache records bounded `cache_read`, `cache_write`, or `invalidation` metrics by failure site. +The shared `encodeRedisFrame`, `decodeRedisFrame`, `decodeTrackedRedisFrame`, and `decodeTrackedRedisReadResult` helpers, the `validateRedisSetReply` and `validateRedisScriptInvalidationReply` reply helpers, the `ceilSupportedCacheTtlMs` TTL guard, and the invalidation Lua source are available from `dialcache/redis-protocol`. A custom write chooses `const createdAtMs = request.createdAtMs === undefined ? Date.now() : request.createdAtMs`, calls `encodeRedisFrame(request.value, createdAtMs)`, and sends one `SET valueKey frame PX cacheTtlMs` after validating the TTL. The fallback clock covers ordinary core writes and direct callers that omit the optional field; supplied values must not be resampled or replaced, and invalid runtime values must still be rejected by the frame encoder. A custom tracked read atomically obtains `[value, watermark]` from the primary. Passing both replies to `decodeTrackedRedisReadResult` opts into discriminated watermark misses; the backward-compatible `decodeTrackedRedisFrame` collapses those misses to `null` and preserves the established `DecodedRedisFrame | null` surface. A missing watermark is treated as zero for valid frames, while a malformed numeric watermark fails closed as a generic miss. Invalidation passes `KEYS = [watermarkKey]` and `ARGV = [futureBufferMs, invalidatedAtMs]`, with one stable client timestamp reused across retries. Custom adapters can throw the root-exported `DialCacheRedisPayloadError`, `DialCacheRedisPayloadEncodingError`, and `DialCacheRedisProtocolError` classes to distinguish malformed replies, unsupported encodings, and reply-domain violations. DialCache records bounded `cache_read`, `cache_write`, or `invalidation` metrics by failure site. Redis values use a compact binary frame: @@ -615,16 +615,16 @@ On a served hit, DialCache returns the already-decoded cached value before start The detached job uses this bounded algorithm: 1. Obtain the original Redis payload as `C0` using the key's existing tracked or untracked read mode. -2. If `C0` is missing, wait for the caller's successfully accepted `S` after its configured fallback boundary. For a typed tracked miss, sample the fill timestamp before serialization: emit `fill_fenced` and stop when that candidate is at or before the observed watermark, without invoking `serializer.dump`, compression, or Redis; otherwise attempt one normal Redis write using the resolved TTL and that exact candidate. For a generic `null` miss, attempt the ordinary write without supplying a core timestamp, preserving adapter-side sampling. Before the whole-job deadline, emit `filled` when Redis accepts the write or `fill_error` when serialization or the write fails. A tracked fill can still be physically stored yet remain fenced if the watermark advances after `C0`. +2. If `C0` is missing, wait for the caller's successfully accepted `S` after its configured fallback boundary. For a discriminated tracked watermark miss, sample a preflight timestamp before serialization: emit `fill_fenced` and stop without invoking `serializer.dump`, compression, or Redis when that timestamp is at or before the observed watermark. Otherwise prepare the serialized payload, sample a final timestamp dispatch-adjacent, and recheck the watermark. Emit `fill_fenced` without dispatching `SET` if the final timestamp is at or before the watermark; otherwise attempt one normal Redis write using the resolved TTL and that exact final timestamp. For a generic `null` miss, attempt the ordinary write without supplying a core timestamp, preserving adapter-side sampling. Before the whole-job deadline, emit `filled` when Redis accepts the write or `fill_error` when serialization or the write fails. A tracked fill can still be physically stored yet remain fenced if the watermark advances after `C0`. 3. If `C0` is non-null, obtain `S`, deserialize an isolated snapshot of `C0`, and run the default or custom semantic comparator. Any non-null `C0` is observation-only: DialCache never repairs or overwrites it, including when deserialization fails. 4. If `C0` and `S` match semantically, emit `match` without another Redis read. 5. Otherwise, reread Redis directly in the same mode as `C1`, bypassing request-local and process-local cache. 6. If `C1` is missing under the normal value/watermark protocol or differs byte-for-byte from `C0`, emit `superseded`; if it is identical, emit `mismatch`. A future-dated `C1` records its offset but remains available for this payload comparison, so a reader-clock step does not change the verdict. 7. If the confirmation read fails or reaches its Redis-read deadline, emit `confirmation_error`. -Here a semantic miss means the Redis read returned `null` or a typed `RedisWatermarkMiss`; it does not include a non-null payload that later fails deserialization. A physical tracked frame rejected by its watermark, timestamp domain, or future-time check is therefore a miss. Only a typed miss carries a trustworthy observed fence and can produce `fill_fenced`; generic or legacy `null` misses retain the normal refill behavior. A caller fallback rejection or timeout never becomes accepted `S` and never starts the fill. +Here a semantic miss means the Redis read returned `null` or `RedisWatermarkMiss { kind: "watermark_miss", observedWatermarkMs }`; it does not include a non-null payload that later fails deserialization. A physical tracked frame rejected by its watermark, timestamp domain, or future-time check is therefore a miss. Only the discriminated miss carries a trustworthy observed fence and can produce `fill_fenced`; generic or legacy `null` misses retain the normal refill behavior. A caller fallback rejection or timeout never becomes accepted `S` and never starts the fill. -Both detached Redis reads use the effective `remoteReadTimeoutMs` and the key's normal native `GET` or `MGET` protocol. The initial `C0` observation enforces the logical remote TTL `F`; the non-serving `C1` confirmation bypasses logical age solely to determine whether the original payload bytes were superseded. Every dispatched semantic-miss fill uses the same serializer, resolved physical TTL, and complete-frame `SET` as the caller path; when stale-on-error is active, it requests physical retention through `M` while serving reads still enforce `F`. A typed watermark miss uses the same exact candidate-timestamp pairing and fence decision as its caller-path counterpart; generic `null` fills preserve the ordinary adapter-side timestamp sample. Tracked `C0` and `C1` reads remain watermark-aware and are routed to primaries by the bundled adapters. The fill itself never issues another watermark read or mutates the watermark; `fill_fenced` is decided from the original `C0` snapshot and adds no round trip. Untracked reads use the ordinary one-key read route, which has no shadow-specific primary guarantee. Strings compare exactly, Buffers compare by bytes, and string/Buffer pairs compare by their UTF-8 bytes. DialCache does not deserialize `C1`, compare it with `S`, or chase another version. +Both detached Redis reads use the effective `remoteReadTimeoutMs` and the key's normal native `GET` or `MGET` protocol. The initial `C0` observation enforces the logical remote TTL `F`; the non-serving `C1` confirmation bypasses logical age solely to determine whether the original payload bytes were superseded. Every dispatched semantic-miss fill uses the same serializer, resolved physical TTL, complete-frame `SET`, and conditional-refill rules as the caller path; generic `null` fills preserve the ordinary adapter-side timestamp sample. Tracked `C0` and `C1` reads remain watermark-aware and are routed to primaries by the bundled adapters. The fill itself never issues another watermark read or mutates the watermark; `fill_fenced` is decided against the watermark from the original `C0` snapshot and adds no round trip. Untracked reads use the ordinary one-key read route, which has no shadow-specific primary guarantee. Strings compare exactly, Buffers compare by bytes, and string/Buffer pairs compare by their UTF-8 bytes. DialCache does not deserialize `C1`, compare it with `S`, or chase another version. The detached scheduler, Redis-read deadline timers, and overall shadow deadline timer are unreferenced, so they do not keep an otherwise idle process alive. Detachment is asynchronous work on the Node event loop, not a worker thread: synchronous source, serializer, or comparator work can still occupy the event loop after the request path has been released. @@ -634,7 +634,7 @@ Detached execution retains the original `cached()` argument references or `getOr Each job has one monotonic deadline across detached `C0`, the SoT result, serializer work, comparison, optional `C1`, and semantic-miss fill. Served-hit timing begins when its detached validation callback starts. On a ramped-down path, timing begins immediately before the caller's SoT invocation so synchronous source work that runs before admission still consumes the same budget. Each Redis read also has its effective read deadline. A finite `fallbackTimeoutMs` is reused as the overall shadow budget. When `fallbackTimeoutMs` is `null`, the normal fallback remains intentionally unbounded, but detached shadow work still uses a 60-second budget. Once timeout delivery marks a job abandoned, DialCache releases retained `C0` references and prevents later phases from starting. -JavaScript promises and Redis writes do not provide a general cancellation or transaction boundary. Work already dispatched may continue and keeps the shadow slot until it settles. A write rejection, `fill_error`, or shadow `timeout` after dispatch does not prove that Redis was unchanged; the command may have executed before its result became unavailable. Conversely, `filled` means the semantic client returned success before the shadow deadline, not that the value is still present. `fill_fenced` is stronger but narrower evidence: DialCache locally skipped dispatch because its candidate timestamp was already at or before the valid watermark observed by `C0`; it does not prove that Redis stayed unchanged afterward. Give dependencies finite native budgets and treat shadow outcomes as best-effort operational evidence. +JavaScript promises and Redis writes do not provide a general cancellation or transaction boundary. Work already dispatched may continue and keeps the shadow slot until it settles. A write rejection, `fill_error`, or shadow `timeout` after dispatch does not prove that Redis was unchanged; the command may have executed before its result became unavailable. Conversely, `filled` means the semantic client returned success before the shadow deadline, not that the value is still present. `fill_fenced` is stronger but narrower evidence: one of DialCache's local fence checks skipped dispatch against the valid watermark observed by `C0`; it does not prove that Redis stayed unchanged afterward. Give dependencies finite native budgets and treat shadow outcomes as best-effort operational evidence. A `match` means application-level value equality: @@ -658,9 +658,9 @@ The byte caps apply before logger framing or escaping, so they do not guarantee Detached Redis reads, serializer loads/dumps, payload sizes, and read/write errors use the existing layer label with `layer="remote_shadow"`. This distinguishes non-serving Redis cost from caller-path `layer="remote"` telemetry without adding a metric or label. The established `observeGet{layer="remote"}` boundary includes caller-path deserialization, while `observeGet{layer="remote_shadow"}` ends when the deadline-bounded Redis read result settles; detached serializer work and any later raw-client settlement are outside that timer. The request-path read that supplied a served `C0` keeps `layer="remote"`, and a ramped-down caller keeps `disabled{layer="remote", reason="ramped_down"}`. No `disabled{layer="remote_shadow"}` event is emitted for ineligible or dropped work; `dropped` remains the terminal shadow outcome. Confirmation reads use the same `remote_shadow` value, with `superseded` or `confirmation_error` describing their role. -The command amplification is bounded: a selected served hit adds one SoT read and adds `C1` only for a semantic mismatch candidate; a selected ramped-down hit adds detached `C0`, reuses the caller's existing SoT read, and likewise adds `C1` only for a candidate; a selected ramped-down miss adds detached `C0` and at most one fill in the key's existing mode. A known-fenced tracked fill stops before serialization and adds no `SET`; no path adds a fence-check command. `superseded` means only that the original observation could not be confirmed. `mismatch` means the exact `C0` payload survived another Redis read after the SoT disagreement; it is not a cross-system atomic snapshot or a guarantee that the mismatch persists. For an untracked key it is also not proof of primary freshness or invalidation safety. +The command amplification is bounded: a selected served hit adds one SoT read and adds `C1` only for a semantic mismatch candidate; a selected ramped-down hit adds detached `C0`, reuses the caller's existing SoT read, and likewise adds `C1` only for a candidate; a selected ramped-down miss adds detached `C0` and at most one fill in the key's existing mode. A fenced tracked fill adds no `SET`, and a preflight fence also avoids payload preparation; no path adds a fence-check command. `superseded` means only that the original observation could not be confirmed. `mismatch` means the exact `C0` payload survived another Redis read after the SoT disagreement; it is not a cross-system atomic snapshot or a guarantee that the mismatch persists. For an untracked key it is also not proof of primary freshness or invalidation safety. -The initial `C0` read and later fill are not atomic. A typed miss can suppress a fill already known to be fenced by the `C0` watermark, but an allowed fill is still a normal overwrite, not a compare-and-set or write-if-still-missing operation: another writer can populate Redis after the semantic miss and be overwritten by the shadow fill, or a later invalidation can fence it. For tracked keys, the next atomic value-and-watermark read still rejects a frame whose client timestamp is at or before the watermark, so size `futureBufferMs` to cover the complete SoT, serialization, client queue, network, write interval, and fleet clock-skew budget when stale-serving protection matters. An untracked shadow fill has no such read fence and retains the ordinary TTL-based last-writer-wins contract; because it is detached, an older accepted source value may be written after a concurrent source mutation and remain until expiry. Shadow mode never repairs a frame returned as a non-null semantic `C0`; a physical frame rejected by normal tracked-read semantics is a miss and may be overwritten with a fresh TTL unless its typed fence still rejects the candidate. Shadow work never invalidates, evicts local state, or changes the value returned to the caller. +The initial `C0` read and later fill are not atomic. A discriminated miss can suppress a fill against the `C0` watermark, but an allowed fill is still a normal overwrite, not a compare-and-set or write-if-still-missing operation: another writer can populate Redis after the semantic miss and be overwritten by the shadow fill, or a later invalidation can fence it. For tracked keys, the next atomic value-and-watermark read still rejects a frame whose client timestamp is at or before the watermark, so size `futureBufferMs` to cover the complete SoT, serialization, client queue, network, write interval, and fleet clock-skew budget when stale-serving protection matters. An untracked shadow fill has no such read fence and retains the ordinary TTL-based last-writer-wins contract; because it is detached, an older accepted source value may be written after a concurrent source mutation and remain until expiry. Shadow mode never repairs a frame returned as a non-null semantic `C0`; a physical frame rejected by normal tracked-read semantics is a miss and may be overwritten with a fresh TTL unless its discriminated fence still rejects the refill. Shadow work never invalidates, evicts local state, or changes the value returned to the caller. A served-hit sample invokes the wrapped function or inline loader as an additional source read, so that loader must be safe to call for observation. A ramped-down sample reuses the caller's ordinary invocation and does not add another SoT call. @@ -721,7 +721,7 @@ Invalidation writes a Redis watermark at `{encodedNamespace:encodedKeyType:encod The internal `:dialcache-frame-v1` suffix identifies values written with DialCache's binary protocol. Watermarks are stored as decimal timestamps. -A cached Redis value whose writer-provided `createdAtMs` is older than or equal to the watermark is a tracked miss. `invalidateRemote(keyType, id, futureBufferMs)` proposes the invalidating process's `Date.now()` plus the buffer, and Lua keeps the greater of that proposal and the existing watermark. A tracked read obtains value and watermark in one primary-routed `MGET`; a missing watermark is the zero baseline, malformed or out-of-range decimal state fails closed, and `createdAtMs <= watermark` misses. Redis also returns `nil` for a wrong-type member of `MGET`, so a wrong-type watermark has the same zero-baseline behavior as an absent one until the next explicit invalidation repairs it. When the same snapshot contains a present valid numeric watermark and an adapter-level semantic miss, the bundled adapter returns `RedisWatermarkMiss { observedWatermarkMs }`. After the fallback succeeds, DialCache samples one candidate timestamp before serialization. If `candidateCreatedAtMs <= observedWatermarkMs`, fallback still returns normally but the known-fenced replacement is skipped before serializer/compression/frame work and `SET`; if it is greater, the exact candidate is encoded in the normal complete-frame write. Missing or malformed watermark metadata and legacy custom adapters that return `null` retain the normal refill behavior. Native `MGET` must still transfer the full stored frame before Node can apply the fence verdict, so a future window can repeatedly transfer a large fenced payload even when replacement work is suppressed. A tracked invocation that reaches the Redis read/write path does not publish its fallback directly to process-local cache; a later validated Redis hit may warm it. Local-only, remote-policy-disabled, and ramped-down paths remain governed by local policy, while request-local memoization remains unconditional. A ramped-out invocation without shadow work does not consult Redis; selected tracked shadow reads remain watermark-aware. +A cached Redis value whose writer-provided `createdAtMs` is older than or equal to the watermark is a tracked miss. `invalidateRemote(keyType, id, futureBufferMs)` proposes the invalidating process's `Date.now()` plus the buffer, and Lua keeps the greater of that proposal and the existing watermark. A tracked read obtains value and watermark in one primary-routed `MGET`; a missing watermark is the zero baseline, malformed or out-of-range decimal state fails closed, and `createdAtMs <= watermark` misses. Redis also returns `nil` for a wrong-type member of `MGET`, so a wrong-type watermark has the same zero-baseline behavior as an absent one until the next explicit invalidation repairs it. When the same snapshot contains a present valid numeric watermark and an adapter-level semantic miss, the bundled adapter returns `RedisWatermarkMiss { kind: "watermark_miss", observedWatermarkMs }`. After the fallback succeeds, DialCache samples a preflight timestamp before serialization. If it is at or before the observed watermark, fallback still returns normally but the known-fenced replacement is skipped before serializer/compression/frame work and `SET`. Otherwise DialCache prepares the payload, samples a final dispatch-adjacent timestamp, and rechecks the same watermark. A final timestamp at or before the watermark suppresses `SET`; an admitted write encodes that exact final timestamp so serialization time does not consume the stored value's logical TTL. Missing or malformed watermark metadata and legacy custom adapters that return `null` retain the normal refill behavior. Native `MGET` must still transfer the full stored frame before Node can apply the fence verdict, so a future window can repeatedly transfer a large fenced payload even when replacement work is suppressed. A tracked invocation that reaches the Redis read/write path does not publish its fallback directly to process-local cache; a later validated Redis hit may warm it. Local-only, remote-policy-disabled, and ramped-down paths remain governed by local policy, while request-local memoization remains unconditional. A ramped-out invocation without shadow work does not consult Redis; selected tracked shadow reads remain watermark-aware. All serving timestamps come from application-process epoch clocks; DialCache does not call Redis `TIME`, estimate an offset, or compensate for skew. Participating application nodes therefore need external clock synchronization and monitoring. Healthy managed node pools commonly stay close, but Kubernetes does not guarantee a maximum offset, and pauses or NTP faults can be much larger than normal millisecond-scale skew. Relative clock differences shift logical expiry early or late, while frames dated after a reader clock fail closed until that clock catches up. Operation durations and deadlines continue to use the monotonic `performance.now()` clock. diff --git a/scripts/test-package.mjs b/scripts/test-package.mjs index 6e4c0db..3397469 100644 --- a/scripts/test-package.mjs +++ b/scripts/test-package.mjs @@ -349,7 +349,8 @@ const decodedTrackedRedisReadResult: RedisReadResult = decodeTrackedRedisReadRes ); if ( decodedTrackedRedisReadResult !== null - && "observedWatermarkMs" in decodedTrackedRedisReadResult + && "kind" in decodedTrackedRedisReadResult + && decodedTrackedRedisReadResult.kind === "watermark_miss" ) { const typedWatermarkMiss: RedisWatermarkMiss = decodedTrackedRedisReadResult; const observedWatermarkMs: number = typedWatermarkMiss.observedWatermarkMs; @@ -1010,7 +1011,11 @@ const esmWatermarkMiss = redisProtocol.decodeTrackedRedisReadResult( redisProtocol.encodeRedisFrame("pending", 0), Buffer.from("0"), ); -if (esmWatermarkMiss?.observedWatermarkMs !== 0 || "payload" in esmWatermarkMiss) { +if ( + esmWatermarkMiss?.kind !== "watermark_miss" + || esmWatermarkMiss.observedWatermarkMs !== 0 + || "payload" in esmWatermarkMiss +) { throw new Error("The packed ESM tracked result decoder did not preserve the observed watermark miss"); } if (redisProtocol.decodeTrackedRedisFrame(redisProtocol.encodeRedisFrame("value", 1), null)?.payload !== "value") { @@ -1395,7 +1400,11 @@ const cjsWatermarkMiss = redisProtocol.decodeTrackedRedisReadResult( redisProtocol.encodeRedisFrame("pending", 0), Buffer.from("0"), ); -if (cjsWatermarkMiss?.observedWatermarkMs !== 0 || "payload" in cjsWatermarkMiss) { +if ( + cjsWatermarkMiss?.kind !== "watermark_miss" + || cjsWatermarkMiss.observedWatermarkMs !== 0 + || "payload" in cjsWatermarkMiss +) { throw new Error("The packed CommonJS tracked result decoder did not preserve the observed watermark miss"); } if (redisProtocol.decodeTrackedRedisFrame(redisProtocol.encodeRedisFrame("value", 1), null)?.payload !== "value") { diff --git a/src/dialcache.ts b/src/dialcache.ts index ea0938c..d0cec68 100644 --- a/src/dialcache.ts +++ b/src/dialcache.ts @@ -24,11 +24,12 @@ import { type MetricLayer, type ShadowValidationOutcome, } from "./metrics.js"; -import type { - DecodedRedisFrame, - RedisCachePayload, - RedisReadResult, - RedisWatermarkMiss, +import { + isRedisWatermarkMiss, + type DecodedRedisFrame, + type RedisCachePayload, + type RedisReadResult, + type RedisWatermarkMiss, } from "./redis-client.js"; import type { Serializer } from "./serializer.js"; import type { CacheGetResult, RemoteCacheGetResult } from "./internal/cache-result.js"; @@ -1773,14 +1774,6 @@ function redisPayloadsEqual(left: RedisCachePayload, right: RedisCachePayload): return Buffer.isBuffer(right) && right.equals(Buffer.from(left, "utf8")); } -function isRedisWatermarkMiss(result: RedisReadResult): result is RedisWatermarkMiss { - return typeof result === "object" - && result !== null - && "observedWatermarkMs" in result - && !("payload" in result) - && !("createdAtMs" in result); -} - async function settleUnexpectedThenable(value: unknown): Promise { if (value === null || (typeof value !== "object" && typeof value !== "function")) { return; diff --git a/src/internal/redis-cache.ts b/src/internal/redis-cache.ts index cf0e0c0..db0b10c 100644 --- a/src/internal/redis-cache.ts +++ b/src/internal/redis-cache.ts @@ -11,12 +11,13 @@ import { type MetricLayer, type StaleRecoveryOutcome, } from "../metrics.js"; -import type { - DecodedRedisFrame, - DialCacheRedisClient, - RedisCachePayload, - RedisReadResult, - RedisWatermarkMiss, +import { + isRedisWatermarkMiss, + type DecodedRedisFrame, + type DialCacheRedisClient, + type RedisCachePayload, + type RedisReadResult, + type RedisWatermarkMiss, } from "../redis-client.js"; import { JsonSerializer, type Serializer } from "../serializer.js"; import type { RedisCacheGetResult } from "./cache-result.js"; @@ -300,18 +301,14 @@ export class RedisCache { const cacheTtlMs = key.trackForInvalidation ? Math.min(configuredTtlMs, MAX_TRACKED_REDIS_VALUE_TTL_MS) : configuredTtlMs; - let createdAtMs: number | undefined; - if (key.trackForInvalidation && watermarkMiss !== undefined) { - createdAtMs = Date.now(); - try { - assertValidRedisTimestampMs(createdAtMs); - } catch (error) { - this.recordError(key, metricLayer, "cache_write"); - throw error; - } - if (createdAtMs <= watermarkMiss.observedWatermarkMs) { - return false; - } + const observedWatermarkMs = key.trackForInvalidation + ? watermarkMiss?.observedWatermarkMs + : undefined; + if ( + observedWatermarkMs !== undefined + && this.sampleWriteTimestamp(key, metricLayer) <= observedWatermarkMs + ) { + return false; } const start = performance.now(); @@ -353,6 +350,13 @@ export class RedisCache { if (shouldWrite !== undefined && !shouldWrite()) { return false; } + let createdAtMs: number | undefined; + if (observedWatermarkMs !== undefined) { + createdAtMs = this.sampleWriteTimestamp(key, metricLayer); + if (createdAtMs <= observedWatermarkMs) { + return false; + } + } if (cacheTtlMs < configuredTtlMs) { this.recordError(key, metricLayer, "tracked_ttl_clamped"); } @@ -579,6 +583,17 @@ export class RedisCache { this.recordMetric((metrics) => metrics.error({ ...labelsFor(key, layer), error: kind, inFallback: false })); } + private sampleWriteTimestamp(key: DialCacheKey, layer: MetricLayer): number { + const createdAtMs = Date.now(); + try { + assertValidRedisTimestampMs(createdAtMs); + } catch (error) { + this.recordError(key, layer, "cache_write"); + throw error; + } + return createdAtMs; + } + private recordStaleRecovery( key: DialCacheKey, outcome: StaleRecoveryOutcome, @@ -612,14 +627,6 @@ function elapsedSeconds(startMs: number): number { return Math.max((performance.now() - startMs) / 1000, 0); } -function isRedisWatermarkMiss(result: RedisReadResult): result is RedisWatermarkMiss { - return typeof result === "object" - && result !== null - && "observedWatermarkMs" in result - && !("payload" in result) - && !("createdAtMs" in result); -} - function isValidRedisWatermarkMiss(miss: RedisWatermarkMiss): boolean { return Number.isSafeInteger(miss.observedWatermarkMs) && miss.observedWatermarkMs >= 0; diff --git a/src/internal/redis-payload.ts b/src/internal/redis-payload.ts index 2cfd953..9123553 100644 --- a/src/internal/redis-payload.ts +++ b/src/internal/redis-payload.ts @@ -1,6 +1,7 @@ import { DialCacheRedisPayloadEncodingError, DialCacheRedisPayloadError, + isRedisWatermarkMiss, type DecodedRedisFrame, type RedisCachePayload, type RedisReadResult, @@ -130,7 +131,10 @@ export function decodeTrackedRedisReadResult( if (watermark === null) { return null; } - return decodeTrackedFrame(frame, watermark, { observedWatermarkMs: watermark }); + return decodeTrackedFrame(frame, watermark, { + kind: "watermark_miss", + observedWatermarkMs: watermark, + }); } /** @@ -164,13 +168,6 @@ function decodeTrackedFrame( }; } -function isRedisWatermarkMiss(result: RedisReadResult): result is RedisWatermarkMiss { - return result !== null - && "observedWatermarkMs" in result - && !("payload" in result) - && !("createdAtMs" in result); -} - function readFrameCreatedAtMs(frame: Buffer): number { return Number(frame.readBigUInt64BE(REDIS_FRAME_TIMESTAMP_OFFSET)); } diff --git a/src/redis-client.ts b/src/redis-client.ts index bc5720b..da65290 100644 --- a/src/redis-client.ts +++ b/src/redis-client.ts @@ -86,6 +86,7 @@ export interface DecodedRedisFrame { * before `observedWatermarkMs` is known to remain unreadable. */ export interface RedisWatermarkMiss { + readonly kind: "watermark_miss"; readonly observedWatermarkMs: number; readonly payload?: never; readonly createdAtMs?: never; @@ -98,6 +99,16 @@ export interface RedisWatermarkMiss { */ export type RedisReadResult = DecodedRedisFrame | RedisWatermarkMiss | null; +/** Package-private runtime discriminator for the semantic miss variant. */ +export function isRedisWatermarkMiss(result: unknown): result is RedisWatermarkMiss { + return typeof result === "object" + && result !== null + && "kind" in result + && result.kind === "watermark_miss" + && !("payload" in result) + && !("createdAtMs" in result); +} + interface RedisValueRequest { readonly valueKey: string; } @@ -127,11 +138,12 @@ export interface RedisWriteRequest extends RedisValueRequest { readonly value: RedisCachePayload; /** * Nonnegative safe-integer epoch milliseconds to encode in the frame. - * DialCache core supplies this for refills following `RedisWatermarkMiss`. + * DialCache core supplies the final dispatch-adjacent sample for admitted + * refills following `RedisWatermarkMiss`. * It remains optional so ordinary refills, existing direct adapter callers, * and custom adapter implementations keep their established behavior. An * adapter that returns `RedisWatermarkMiss` must honor a supplied value - * exactly so the refill decision and stored frame cannot diverge. + * exactly so the final fence decision and stored frame cannot diverge. */ readonly createdAtMs?: number; } @@ -182,7 +194,8 @@ export interface DialCacheRedisClient { * miss when the same snapshot contained a present, valid numeric watermark. * Existing adapters may continue returning `null` and remain correct while * missing the conditional refill optimization. Adapters that opt into the - * typed miss must also honor `RedisWriteRequest.createdAtMs` when supplied. + * discriminated miss must also honor `RedisWriteRequest.createdAtMs` when + * supplied. * * A returned frame's payload is transferred to DialCache. A returned Buffer * must remain stable and must not be mutated, pooled, or reused after this diff --git a/test/dialcache-invalidation.test.ts b/test/dialcache-invalidation.test.ts index fc89ea3..ea3f0e3 100644 --- a/test/dialcache-invalidation.test.ts +++ b/test/dialcache-invalidation.test.ts @@ -147,6 +147,7 @@ describe("DialCache targeted invalidation watermarks", () => { valueKey: valueKey("FutureBufferUser"), watermarkKey, })).resolves.toEqual({ + kind: "watermark_miss", observedWatermarkMs: Date.parse("2026-05-12T18:00:01.000Z"), }); await expect(redis.read({ valueKey: valueKey("FutureBufferUser") })).resolves.toBeNull(); @@ -178,6 +179,81 @@ describe("DialCache targeted invalidation watermarks", () => { expect(decodeFrame(redis.raw(valueKey("NewerRefillCandidate"))).createdAtMs).toBe(now); }); + it("starts an admitted refill's logical TTL after slow serialization", async () => { + const now = Date.now(); + const useCase = "PostSerializationRefillCandidate"; + const redis = new FakeRedis(); + redis.setRaw(valueKey(useCase), encodeFrame({ source: "old" }, now - 15)); + redis.setRaw(watermarkKey, String(now - 5)); + const serializer: Serializer<{ userId: string; calls: number }> = { + dump: vi.fn(async (value) => { + vi.advanceTimersByTime(1_500); + return JSON.stringify(value); + }), + load: vi.fn((value) => { + const payload = Buffer.isBuffer(value) ? value.toString("utf8") : value; + return JSON.parse(payload) as { userId: string; calls: number }; + }), + }; + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); + let calls = 0; + const getUser = dialcache.cached(async (userId: string) => ({ userId, calls: ++calls }), { + keyType: "user_id", + useCase, + cacheKey: (userId) => userId, + trackForInvalidation: true, + defaultConfig: remoteOnly(1), + serializer, + }); + + const first = await dialcache.enable(async () => await getUser("123")); + const second = await dialcache.enable(async () => await getUser("123")); + + expect(first).toEqual({ userId: "123", calls: 1 }); + expect(second).toEqual(first); + expect(calls).toBe(1); + expect(redis.mGetCalls).toBe(2); + expect(redis.setCalls).toBe(1); + expect(decodeFrame(redis.raw(valueKey(useCase))).createdAtMs).toBe(now + 1_500); + }); + + it("suppresses an admitted refill when the clock rolls behind the watermark during serialization", async () => { + const now = Date.now(); + const observedWatermarkMs = now - 1; + const useCase = "RolledBackRefillCandidate"; + const redis = new FakeRedis(); + redis.setRaw(valueKey(useCase), encodeFrame({ source: "old" }, now - 15)); + redis.setRaw(watermarkKey, String(observedWatermarkMs)); + const dump = vi.fn(async (value: { userId: string; calls: number }) => { + vi.setSystemTime(observedWatermarkMs - 1); + return JSON.stringify(value); + }); + const serializer: Serializer<{ userId: string; calls: number }> = { + dump, + load: () => { + throw new Error("fenced stale frame must not deserialize"); + }, + }; + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); + let calls = 0; + const getUser = dialcache.cached(async (userId: string) => ({ userId, calls: ++calls }), { + keyType: "user_id", + useCase, + cacheKey: (userId) => userId, + trackForInvalidation: true, + defaultConfig: remoteOnly(), + serializer, + }); + + const value = await dialcache.enable(async () => await getUser("123")); + + expect(value).toEqual({ userId: "123", calls: 1 }); + expect(dump).toHaveBeenCalledOnce(); + expect(redis.mGetCalls).toBe(1); + expect(redis.setCalls).toBe(0); + expect(decodeFrame(redis.raw(valueKey(useCase))).createdAtMs).toBe(now - 15); + }); + it.each([ { boundary: "equal to", watermarkOffsetMs: 0 }, { boundary: "behind", watermarkOffsetMs: 10 }, @@ -248,6 +324,7 @@ describe("DialCache targeted invalidation watermarks", () => { valueKey: valueKey("FutureBufferFallbackRace"), watermarkKey, })).resolves.toEqual({ + kind: "watermark_miss", observedWatermarkMs: Date.parse("2026-05-12T18:00:01.000Z"), }); }); @@ -302,6 +379,7 @@ describe("DialCache targeted invalidation watermarks", () => { valueKey: valueKey("FutureBufferSerializationRace"), watermarkKey, })).resolves.toEqual({ + kind: "watermark_miss", observedWatermarkMs: Date.parse("2026-05-12T18:00:01.000Z"), }); }); diff --git a/test/dialcache-redis.test.ts b/test/dialcache-redis.test.ts index 15d9e8d..44fafec 100644 --- a/test/dialcache-redis.test.ts +++ b/test/dialcache-redis.test.ts @@ -9,6 +9,7 @@ import { type DialCacheMetricsAdapter, type DialCacheRedisClient, type RedisConfig, + type RedisWriteRequest, type Serializer, } from "../src/index.js"; import { decodeFrame, encodeFrame, FakeRedis } from "./fake-redis.js"; @@ -269,7 +270,7 @@ describe("DialCache Redis TTL layer", () => { expect(metrics.miss).toHaveBeenCalledOnce(); }); - it("preserves structurally compatible custom frames with extra watermark metadata", async () => { + it("preserves custom frames that collide with miss metadata", async () => { const nowMs = 1_700_000_000_000; vi.spyOn(Date, "now").mockReturnValue(nowMs); const cachedValue = { source: "redis" }; @@ -277,6 +278,7 @@ describe("DialCache Redis TTL layer", () => { read: vi.fn(async () => ({ payload: JSON.stringify(cachedValue), createdAtMs: nowMs, + kind: "watermark_miss", observedWatermarkMs: nowMs - 1, })), write: vi.fn(async () => undefined), @@ -308,6 +310,59 @@ describe("DialCache Redis TTL layer", () => { expect(redis.write).not.toHaveBeenCalled(); }); + it.each([ + { name: "NaN watermark", trackForInvalidation: true, observedWatermarkMs: Number.NaN }, + { name: "negative watermark", trackForInvalidation: true, observedWatermarkMs: -1 }, + { name: "fractional watermark", trackForInvalidation: true, observedWatermarkMs: 1.5 }, + { + name: "unsafe watermark", + trackForInvalidation: true, + observedWatermarkMs: Number.MAX_SAFE_INTEGER + 1, + }, + { + name: "typed miss on an untracked request", + trackForInvalidation: false, + observedWatermarkMs: 1_700_000_001_000, + }, + ])("normalizes $name to an ordinary miss", async ({ trackForInvalidation, observedWatermarkMs }) => { + vi.spyOn(Date, "now").mockReturnValue(1_700_000_000_000); + const write = vi.fn(async (_request: RedisWriteRequest) => undefined); + const redis: DialCacheRedisClient = { + read: vi.fn(async () => ({ + kind: "watermark_miss" as const, + observedWatermarkMs, + })), + write, + invalidate: vi.fn(async () => undefined), + }; + const serializer: Serializer<{ readonly source: string }> = { + dump: vi.fn((value) => JSON.stringify(value)), + load: vi.fn(() => { + throw new Error("invalid typed misses must not be deserialized"); + }), + }; + const fallback = vi.fn(async () => ({ source: "fallback" })); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); + const getUser = dialcache.cached(fallback, { + keyType: "user_id", + useCase: "RedisInvalidWatermarkMiss", + cacheKey: () => "123", + trackForInvalidation, + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + ramp: { [CacheLayer.REMOTE]: 100 }, + }), + serializer, + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual({ source: "fallback" }); + + expect(serializer.load).not.toHaveBeenCalled(); + expect(fallback).toHaveBeenCalledOnce(); + expect(write).toHaveBeenCalledOnce(); + expect(write.mock.calls[0]?.[0]).not.toHaveProperty("createdAtMs"); + }); + it.each([ Number.NaN, Number.POSITIVE_INFINITY, diff --git a/test/dialcache-shadow-confirmation.test.ts b/test/dialcache-shadow-confirmation.test.ts index 134d92d..612c499 100644 --- a/test/dialcache-shadow-confirmation.test.ts +++ b/test/dialcache-shadow-confirmation.test.ts @@ -87,7 +87,8 @@ function isWatermarkMiss(result: ScriptedReadResult): result is RedisWatermarkMi return typeof result === "object" && !Buffer.isBuffer(result) && result !== null - && "observedWatermarkMs" in result; + && "kind" in result + && result.kind === "watermark_miss"; } type OrdinaryMetricName = @@ -616,7 +617,7 @@ describe("DialCache Redis shadow confirmation", () => { const frameCreatedAtMs = Date.now(); const redis = new ScriptedRedis([ () => payload, - () => ({ observedWatermarkMs: frameCreatedAtMs + 1 }), + () => ({ kind: "watermark_miss", observedWatermarkMs: frameCreatedAtMs + 1 }), ]); redis.frameCreatedAtMs = frameCreatedAtMs; const metrics = new RecordingMetrics(); @@ -1398,7 +1399,7 @@ describe("DialCache Redis shadow confirmation", () => { const nowMs = 1_700_000_000_000; const nowSpy = vi.spyOn(Date, "now").mockReturnValue(nowMs); try { - const redis = new ScriptedRedis([() => ({ observedWatermarkMs: nowMs })]); + const redis = new ScriptedRedis([() => ({ kind: "watermark_miss", observedWatermarkMs: nowMs })]); const metrics = new RecordingMetrics(); const serializer: Serializer<{ readonly id: string }> = { dump: vi.fn(() => { @@ -1440,6 +1441,47 @@ describe("DialCache Redis shadow confirmation", () => { } }); + it("fills a typed shadow miss when the final candidate clears the observed watermark", async () => { + const nowMs = 1_700_000_000_000; + const nowSpy = vi.spyOn(Date, "now").mockReturnValue(nowMs); + try { + const redis = new ScriptedRedis([() => ({ + kind: "watermark_miss", + observedWatermarkMs: nowMs - 1, + })]); + const metrics = new RecordingMetrics(); + const serializer: Serializer<{ readonly id: string }> = { + dump: vi.fn((value) => JSON.stringify(value)), + load: vi.fn(() => { + throw new Error("watermark miss must not deserialize"); + }), + }; + const source = vi.fn(async () => ({ id: "123" })); + const dialcache = createCache(redis, metrics); + const getUser = dialcache.cached(source, { + ...trackedOptions("ShadowDarkFillAboveWatermark", remoteConfig(0)), + cacheKey: () => "123", + serializer, + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual({ id: "123" }); + await waitForShadowEvents(metrics, 1); + + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["filled"]); + expect(source).toHaveBeenCalledOnce(); + expect(serializer.dump).toHaveBeenCalledOnce(); + expect(serializer.load).not.toHaveBeenCalled(); + expect(redis.write).toHaveBeenCalledOnce(); + expect(redis.write).toHaveBeenCalledWith(expect.objectContaining({ + value: JSON.stringify({ id: "123" }), + createdAtMs: nowMs, + })); + expectTrackedReads(redis, 1); + } finally { + nowSpy.mockRestore(); + } + }); + it("reports a detached serializer dump failure as fill_error with an exact remote_shadow error", async () => { const redis = new ScriptedRedis([() => null]); const metrics = new RecordingMetrics(); diff --git a/test/fake-redis.ts b/test/fake-redis.ts index e443b32..6e51240 100644 --- a/test/fake-redis.ts +++ b/test/fake-redis.ts @@ -130,7 +130,7 @@ export class FakeRedis implements DialCacheRedisClient { return null; } if (watermark !== null) { - watermarkMiss = { observedWatermarkMs: watermark }; + watermarkMiss = { kind: "watermark_miss", observedWatermarkMs: watermark }; } } diff --git a/test/node-redis.test.ts b/test/node-redis.test.ts index edf639a..24c1113 100644 --- a/test/node-redis.test.ts +++ b/test/node-redis.test.ts @@ -135,7 +135,7 @@ describe("node-redis adapter", () => { await expect(adapter.read({ valueKey: "tracked:{id}:value", watermarkKey: "tracked:{id}:watermark", - })).resolves.toEqual({ observedWatermarkMs: 1_234 }); + })).resolves.toEqual({ kind: "watermark_miss", observedWatermarkMs: 1_234 }); expect(client.sendCommand).toHaveBeenCalledTimes(1); }); diff --git a/test/redis-cluster.integration.test.ts b/test/redis-cluster.integration.test.ts index dea2967..fd3732d 100644 --- a/test/redis-cluster.integration.test.ts +++ b/test/redis-cluster.integration.test.ts @@ -295,6 +295,7 @@ describe("DialCache Redis protocol on Redis Cluster", () => { const observedWatermark = await cluster.get(watermarkKey); expect(observedWatermark).not.toBeNull(); expect(await adapter.read({ valueKey, watermarkKey })).toEqual({ + kind: "watermark_miss", observedWatermarkMs: Number(observedWatermark), }); await expect( @@ -338,6 +339,7 @@ describe("DialCache Redis protocol on Redis Cluster", () => { const observedWatermark = await cluster.get(watermarkKey); expect(observedWatermark).not.toBeNull(); expect(await adapter.read({ valueKey, watermarkKey })).toEqual({ + kind: "watermark_miss", observedWatermarkMs: Number(observedWatermark), }); }); diff --git a/test/redis-payload.test.ts b/test/redis-payload.test.ts index 82b0b2e..0b91eb8 100644 --- a/test/redis-payload.test.ts +++ b/test/redis-payload.test.ts @@ -96,6 +96,7 @@ describe("Redis frame decoding", () => { encodeFrame("fenced", 0, 1_000), ]) { expect(decodeTrackedRedisReadResult(frame, watermark)).toEqual({ + kind: "watermark_miss", observedWatermarkMs: 1_000, }); expect(decodeTrackedRedisFrame(frame, watermark)).toBeNull(); diff --git a/test/redis-real.integration.test.ts b/test/redis-real.integration.test.ts index c221908..028b788 100644 --- a/test/redis-real.integration.test.ts +++ b/test/redis-real.integration.test.ts @@ -1184,6 +1184,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(invalidate).not.toHaveBeenCalled(); expect(await admin.exists(valueKey)).toBe(0); expect(await client.adapter.read({ valueKey, watermarkKey })).toEqual({ + kind: "watermark_miss", observedWatermarkMs: candidateAtMs, }); expect(await admin.get(watermarkKey)).toBe(String(candidateAtMs)); @@ -1230,6 +1231,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(await admin.scriptExists(INVALIDATE_CACHE_SHA1)).toEqual([true]); const watermark = Number(await admin.get(watermarkKey)); expect(await scriptClient.read({ valueKey: trackedValueKey, watermarkKey })).toEqual({ + kind: "watermark_miss", observedWatermarkMs: watermark, }); }); @@ -1261,6 +1263,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { await admin.set(watermarkKey, "1000"); expect(await scriptClient.read({ valueKey, watermarkKey })).toEqual({ + kind: "watermark_miss", observedWatermarkMs: 1_000, }); @@ -1342,6 +1345,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { await admin.set(watermarkKey, "0"); await expect(scriptClient.read({ valueKey })).rejects.toThrow(/WRONGTYPE/); await expect(scriptClient.read({ valueKey, watermarkKey })).resolves.toEqual({ + kind: "watermark_miss", observedWatermarkMs: 0, }); @@ -1367,6 +1371,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(await admin.pTTL(watermarkKey)).toBeGreaterThan(MIN_WATERMARK_TTL_MS - 1_000); expect(await admin.pTTL(watermarkKey)).toBeLessThanOrEqual(MIN_WATERMARK_TTL_MS); expect(await scriptClient.read({ valueKey, watermarkKey })).toEqual({ + kind: "watermark_miss", observedWatermarkMs: invalidatedAtMs + 100, }); @@ -1810,6 +1815,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { await scriptClient.invalidate({ watermarkKey, futureBufferMs: 100 }); expect(await scriptClient.read({ valueKey, watermarkKey })).toEqual({ + kind: "watermark_miss", observedWatermarkMs: invalidatedAtMs + 100, }); const watermarkBeforeWrite = await admin.get(watermarkKey); @@ -1817,6 +1823,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { await scriptClient.write({ ...writeRequest, value: "behind-watermark" }); expect((await scriptClient.read({ valueKey }))?.payload).toBe("behind-watermark"); expect(await scriptClient.read({ valueKey, watermarkKey })).toEqual({ + kind: "watermark_miss", observedWatermarkMs: invalidatedAtMs + 100, }); expect(await admin.get(watermarkKey)).toBe(watermarkBeforeWrite); @@ -1847,6 +1854,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { await scriptClient.invalidate({ watermarkKey, futureBufferMs: 60_000 }); const watermark = Number(await admin.get(watermarkKey)); expect(await scriptClient.read({ valueKey, watermarkKey })).toEqual({ + kind: "watermark_miss", observedWatermarkMs: watermark, }); diff --git a/test/valkey-glide.test.ts b/test/valkey-glide.test.ts index 56f7a2d..8951f1e 100644 --- a/test/valkey-glide.test.ts +++ b/test/valkey-glide.test.ts @@ -182,7 +182,7 @@ describe("Valkey GLIDE adapter", () => { await expect(adapter.read({ valueKey: "tracked:{id}:value", watermarkKey: "tracked:{id}:watermark", - })).resolves.toEqual({ observedWatermarkMs: 1_234 }); + })).resolves.toEqual({ kind: "watermark_miss", observedWatermarkMs: 1_234 }); expect(client.exec).toHaveBeenCalledTimes(1); });