diff --git a/AGENTS.md b/AGENTS.md index acc2626..7d05318 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,11 +18,11 @@ src/ prometheus.ts # Optional Prometheus adapter datadog.ts # Optional Datadog (DogStatsD) adapter redis-client.ts # Client-independent semantic Redis interface and its public error classes - node-redis.ts # node-redis adapter and script registration + node-redis.ts # node-redis adapter and invalidation dispatch valkey-glide.ts # Valkey GLIDE adapter (standalone and cluster) redis-protocol.ts # Public frame codec and Lua protocol exports serializer.ts # Serializer contract and JSON implementation - internal/ # Cache layers, runtime config, payload compression, and mutation Lua scripts + internal/ # Cache layers, runtime config, payload compression, and invalidation Lua script test/ # Unit and Redis integration tests ``` @@ -36,10 +36,12 @@ test/ # Unit and Redis integration tests - Cache plumbing fails open; explicit maintenance operations surface mutation failures. - Tracked Redis values and invalidation watermarks share a Redis Cluster hash tag. - Tracked reads run on primaries so replica lag cannot hide invalidation. -- A tracked write's placeholder frame (version byte 0) is unreadable on both - read paths until the stamp script promotes it, and the stamp promotes only - the placeholder carrying its own per-write nonce. -- A SET failure is the tracked write's outcome even when the stamp settled. +- Every Redis value write is one native `SET` of a complete version-1 frame + stamped from the writer process's clock; Redis value writes never create or + extend watermarks. +- A tracked read atomically reads the value and watermark from the primary and + serves the frame only when `createdAtMs` is strictly greater than the + watermark. A missing watermark is the zero baseline. - Local entries are process-local and are not synchronously invalidated across instances. ## Conventions diff --git a/README.md b/README.md index 405f0e6..65fac64 100644 --- a/README.md +++ b/README.md @@ -82,8 +82,8 @@ request-local cache -> process-local cache -> Redis cache -> fallback function - Results from the lower chain are memoized request-locally when that layer is enabled. - Process-local hits return immediately. - Process-local misses try Redis and populate the process-local cache on a Redis hit. -- Redis misses call the fallback and attempt to populate Redis and, when active, the process-local cache. Tracked invalidation may suppress both publications. -- Selected Redis keys can execute non-serving [shadow work](#shadow-validation) that validates hits and fills clean misses, even before Redis is allowed to serve callers. +- Redis misses call the fallback and attempt to write one complete Redis frame. An active untracked process-local layer may publish that fallback result directly. For tracked keys, invocations that reach the Redis read/write path suppress direct process-local publication after fallback until a later validated Redis hit can warm it; local-only, remote-policy-disabled, and ramped-down paths remain governed by their local policy and may publish locally. +- Selected Redis keys can execute non-serving [shadow work](#shadow-validation) that validates hits and fills semantic misses, even before Redis is allowed to serve callers. - Redis read failures and timeouts are logged, counted in metrics, and fail open without attempting a second Redis operation. Redis write failures also fail open. `invalidateRemote` requires a configured Redis client; missing configuration and Redis failures are logged, counted, and rethrown so callers do not assume invalidation succeeded. - Cache-key construction and config-provider failures also fail open and run the fallback uncached. - A missing effective process-local/Redis TTL disables that layer by policy; a configured TTL with no ramp defaults to 100%. Disabled layers record a disabled reason and fall through to the next layer/fallback. @@ -265,7 +265,7 @@ const getUser = dialcache.cached((userId: string) => db.fetchUser(userId), { `ramp` values are percentages from 0 to 100. `0` disables the layer, `100` enables it, and intermediate values are deterministically sampled by cache key and layer, so the same key is consistently sampled in or out of a partial rollout across calls and instances. The assignment algorithm is owned by DialCache and remains stable across releases. Applications that need an externally coordinated cohort can use `cacheConfigProvider` to return a sparse per-key ramp override of `0` or `100`. DialCache fetches and resolves one config snapshot per enabled invocation. Provider errors do not activate defaults: they fail open, record `config_error`, and execute the fallback function uncached. -`shadow.ramp` uses the same inclusive 0–100 percentage domain but is independent of cache-layer serving ramps. Omission and `0` disable shadow work; `100` selects every eligible Redis key; intermediate values assign each exact cache key to a stable shadow cohort across calls and instances. A valid remote policy can therefore use `ramp.remote: 0` with a nonzero `shadow.ramp` to exercise and populate Redis without serving from it. A nonzero value explicitly authorizes detached writes after clean shadow-only misses: tracked keys use their watermark-aware write, while untracked keys use their ordinary TTL-based last-writer-wins write. It does not create another `CacheLayer`, activate Redis without a valid remote TTL, or make a request-local/process-local hit continue to Redis. +`shadow.ramp` uses the same inclusive 0–100 percentage domain but is independent of cache-layer serving ramps. Omission and `0` disable shadow work; `100` selects every eligible Redis key; intermediate values assign each exact cache key to a stable shadow cohort across calls and instances. A valid remote policy can therefore use `ramp.remote: 0` with a nonzero `shadow.ramp` to exercise and populate Redis without serving from it. A nonzero value explicitly authorizes detached native writes after semantic shadow misses; later tracked reads remain watermark-aware, while untracked reads retain ordinary TTL-based last-writer-wins behavior. It does not create another `CacheLayer`, activate Redis without a valid remote TTL, or make a request-local/process-local hit continue to Redis. Remote serving and shadow sampling use independent deterministic cohorts. Equal partial percentages do not imply the same keys, so a partial shadow cohort does not guarantee that every key admitted by a later partial serving ramp was warmed or validated. Use `shadow: { ramp: 100 }` when every otherwise eligible invocation must exercise the non-serving Redis path before a serving-ramp increase. @@ -326,16 +326,15 @@ The limit counts entries rather than estimating JavaScript object memory. Recent ### Redis-backed TTL cache -The Redis layer supports standalone Redis, Valkey, and Redis Cluster. Register DialCache's bundled node-redis scripts when creating the client, then pass that client to DialCache: +The Redis layer supports standalone Redis, Valkey, and Redis Cluster. Connect the underlying client, then wrap it with the bundled adapter: ```ts import { createClient } from "redis"; import { DialCache } from "dialcache"; -import { createNodeRedisDialCacheClient, dialcacheRedisScripts } from "dialcache/node-redis"; +import { createNodeRedisDialCacheClient } from "dialcache/node-redis"; const redisClient = createClient({ url: process.env.REDIS_URL, - scripts: dialcacheRedisScripts, disableOfflineQueue: true, commandsQueueMaxLength: 1_000, socket: { connectTimeout: 2_000 }, @@ -358,7 +357,7 @@ async function shutdown(): Promise { } ``` -`redis.client` is required when Redis is configured and accepts the semantic `DialCacheRedisClient` interface. `redis.readTimeoutMs` is optional and sets the instance default for remote reads; omit it to use 50 ms. Create and connect the underlying client before constructing `DialCache`. Node-redis users should register the supplied mutation scripts and wrap their client with `createNodeRedisDialCacheClient` as shown above; the adapter performs reads with native commands. The registered `dialcache*` methods are DialCache's wiring, not a write API: they return raw script replies — the stamp's `2` means the placeholder was lost, not success — so code invoking them directly must map stamp replies through `resolveTrackedRedisWriteReply` from `dialcache/redis-protocol`. The helper requires node-redis's promise API and does not support `legacyMode`, whose callback surface and `.v4` view do not expose the complete native-command-plus-custom-script contract together. +`redis.client` is required when Redis is configured and accepts the semantic `DialCacheRedisClient` interface. `redis.readTimeoutMs` is optional and sets the instance default for remote reads; omit it to use 50 ms. Create and connect the underlying client before constructing `DialCache`. Node-redis users wrap that connected client with `createNodeRedisDialCacheClient` as shown above; the adapter issues native reads and writes and manages invalidation's `EVALSHA`/`EVAL` dispatch internally. The helper requires node-redis's promise API and does not support `legacyMode`, whose callback surface is incompatible with the required promise-based binary-command contract. Valkey GLIDE users pass an already-created standalone or cluster client and its module namespace to the GLIDE adapter: @@ -389,8 +388,9 @@ function shutdown(): void { Pass the same GLIDE 2.x module namespace that created the client. The adapter uses that namespace's `GlideClient` and `GlideClusterClient` identities, -`Batch` and `ClusterBatch` constructors, and `Decoder.Bytes` without importing -a GLIDE runtime itself. The helper accepts a direct official client instance and +the standalone `Batch` constructor, and `Decoder.Bytes` without importing a +GLIDE runtime itself. Cluster reads route `MGET` directly and do not require +`ClusterBatch`. The helper accepts a direct official client instance and fails during construction when the client came from another module instance or is hidden behind a forwarding wrapper, because it cannot safely infer that wrapper's topology. Custom wrappers can implement `DialCacheRedisClient` @@ -400,19 +400,19 @@ The application owns the complete Redis lifecycle. It creates and connects the u Awaiting those public promises does not drain detached shadow work. Shadow scheduling and deadline timers are unreferenced and completion is not guaranteed during shutdown; Redis operations, source reads, serializers, and asynchronous telemetry already started by shadow work remain caller-owned and may still be active. Stop new work before closing their dependencies and accept that an in-flight shadow fill may have been dispatched even if its final outcome is lost during teardown. DialCache does not add a shutdown hook or keep the process alive to deliver best-effort outcomes. -Neither adapter owns additional resources: both dispatch their mutation scripts by source SHA1 and hold no native handles, so the application simply closes the underlying client after draining work. Applications that construct their own GLIDE `Script` objects should know that on GLIDE 2.0.0, releasing a handle has been observed to break other live handles for the same script source despite GLIDE's documented reference counting. +Neither adapter owns additional resources or native script handles, so the application simply closes the underlying client after draining work. -Reads use native `GET` for untracked entries and one atomic `MGET` for each tracked value-and-watermark pair. The adapters validate and decode the returned frame in the Node process. Tracked reads are deliberately routed to primaries so a lagging replica cannot hide an invalidation watermark. +Reads use native `GET` for untracked entries and one atomic `MGET` for each tracked value-and-watermark pair. The adapters validate and decode the returned frame in the Node process. Tracked reads are deliberately routed to primaries so a lagging replica cannot hide an invalidation watermark. After a tracked read settles, DialCache samples `readerNowMs = Date.now()` once. On caller-serving reads and initial shadow observations, a frame with `createdAtMs > readerNowMs` fails closed as a miss before deserialization and emits the bounded offset observation described under [Metrics](#metrics). A confirmation read still observes a future offset, but payload equality—not a second wall-clock sample—decides whether the original observation was superseded. Untracked reads do not consult the informational frame timestamp for serving; Redis physical TTL remains their normal serving lifetime. -Writes are native too, so the payload never crosses the Redis-to-Lua boundary. An untracked write is one `SET` of a client-encoded frame. A tracked write pipelines two ordered commands on one connection: a `SET` of a version-0 placeholder frame carrying a fresh per-write nonce, then the small payload-free `WRITE_TRACKED_STAMP_SCRIPT`, which fences against the watermark, promotes exactly the placeholder carrying its nonce to a served frame with Redis server time, and maintains the watermark TTL. A placeholder is unreadable on both read paths until promoted, so an interleaved or lost stamp degrades to a miss that expires with the value TTL rather than partial state — including briefly blanking a previously readable key the write replaces. The nonce means the stamp can never revive a frame it does not own: if its paired `SET` was rejected, overwritten, or expired, the stamp reports the placeholder gone and the write fails with `DialCacheRedisPlaceholderLostError` as a `cache_write` error instead of publishing another write's leftovers. Losing a same-key write race is one such outcome, so `cache_write` carries a benign, self-healing floor that concentrates on hot tracked keys at TTL expiry — size write-error alerts for it. The `cache_write` metric itself stays one bounded counter; the error's class and name distinguish the lost-placeholder case in logs, and in the `catch` blocks of code that calls an adapter's `write()` directly — DialCache's own request paths absorb it fail-open rather than rethrowing to callers. Each occurrence also emits one warn through the configured logger (the default is `console`), so fleets expecting hot-key write contention should supply a logger that rate-limits or filters that class. A `SET` failure is the write's outcome even when the stamp settled. The pair is deliberately not a `MULTI`/`EXEC` transaction, which would consume caller-owned `WATCH` state. +Writes are native too, so the payload never crosses the Redis-to-Lua boundary. Each bundled adapter samples the writer process's `Date.now()` once immediately before dispatch and issues one `SET valueKey frame PX cacheTtlMs` containing the complete version-1 frame. Tracked and untracked writes have the same adapter request and command shape; only tracked reads include the watermark key. Core caps a tracked Redis value's physical TTL at one hour, while untracked Redis and local TTLs retain their configured limits. Each dispatched write whose configured tracked TTL exceeds that cap emits `error="tracked_ttl_clamped"`; the write is attempted with the capped TTL. The write never reads, creates, or extends a watermark. A frame written behind an active watermark remains physically present but is a tracked miss until its `createdAtMs` is greater than the watermark. Same-key writes are ordinary Redis last-writer-wins operations, with no Lua, pipeline, or transaction on the write path. -Native commands retain Redis's wrong-type behavior. An untracked `GET` surfaces `WRONGTYPE`; tracked `MGET` represents a wrong-type member as a missing value. A wrong-type tracked value is therefore a clean miss and may be replaced with a valid DialCache frame after the fallback succeeds, while a wrong-type watermark prevents the tracked write from succeeding: its paired `SET` still lands, leaving only an unreadable placeholder until expiry or a later successful write. +Native commands retain Redis's wrong-type behavior. An untracked `GET` surfaces `WRONGTYPE`; tracked `MGET` represents a wrong-type member as a missing value. A wrong-type tracked value is therefore a semantic miss and may be replaced with a valid complete frame after fallback. A wrong-type watermark is indistinguishable from an absent watermark to `MGET`, so tracked reads apply the zero baseline; it does not prevent or alter native value writes. The next explicit invalidation replaces that wrong-type key with a valid string watermark. Other script read failures remain errors and cannot bypass the monotonic update. Node-redis forces tracked cluster commands to the slot primary. GLIDE uses an explicit primary route in cluster mode; in standalone mode it sends `MGET` through a one-command non-atomic batch because direct read commands follow the client's replica-read preference. Standalone batches use the primary, and `MGET` itself provides the atomic snapshot without consuming caller-owned `WATCH` state. The GLIDE helper distinguishes those modes from the direct client's runtime identity and rejects ambiguous clients instead of silently choosing a route. -For the stamp and invalidation scripts, node-redis computes each script's SHA, uses `EVALSHA`, and retries with `EVAL` after `NOSCRIPT`; its cluster client routes commands by their first key and performs that fallback on the selected shard. That retry likewise extends the unreadable-placeholder gap of a tracked write by one round trip on a cold script cache. The GLIDE adapter batches the tracked write's `SET` with an `EVALSHA` of the stamp script — routing cluster write batches to the slot primary — and recovers from a flushed script cache by re-sending the stamp as `EVAL` with its source, which the server caches under the same SHA1, so the first tracked write against a cold script cache pays one extra round trip. A late stamp stays paired to its own placeholder through the nonce; if the placeholder is gone by then, the write fails rather than publishing. Invalidation dispatches as `EVALSHA` by the script's source SHA1 on both adapters, and both retry a rejected dispatch once by re-sending the source as `EVAL`: the invalidation script is idempotent — its watermark only advances and its TTL only widens — so a duplicate execution after an ambiguous failure is harmless, and the retry heals a flushed script cache and an `EVALSHA`-rejecting proxy without depending on error wording. Reply-domain violations are deterministic and are not retried. When the retry also fails, the surfaced error is the retry's. On GLIDE the original rejection is attached as the retry error's `cause` unless it already carries one, and a failing invalidation is bounded by roughly two `requestTimeout` windows. On node-redis the retry rejection surfaces unmodified and the original is discarded — the library rejects every command flushed by a single disconnect with one shared error instance, so the adapter never mutates it — and no per-command deadline exists: `disableOfflineQueue`, `commandsQueueMaxLength`, and `reconnectStrategy` bound queueing and dispatch (the setup snippet above disables the offline queue, which makes a disconnected retry fail fast instead of waiting for reconnect), but a command already written to a hung connection has no reply deadline. Its own `NOSCRIPT` recovery may also add one round trip before the adapter's retry. A retry that heals is indistinguishable from a first-attempt success in DialCache's metrics and logs. The genuinely silent regime is invalidation-dispatch healing: watch server-side `INFO commandstats` for `cmdstat_eval` calls rising in step with invalidation volume while `cmdstat_evalsha` stays flat (a proxy rejecting `EVALSHA` before it reaches Redis) or accrues `rejected_calls` (an ACL denial). A sustained stamp fault is loud by contrast — the ACL paragraph below describes its amplitude. +Invalidation is the only remaining Lua operation. Both adapters dispatch it as `EVALSHA` by the script source's SHA1 and retry a rejected dispatch once by re-sending the source as `EVAL`. The script is idempotent: its watermark advances monotonically and its TTL only widens, so a duplicate execution after an ambiguous failure is harmless. Reply-domain violations are deterministic and are not retried. If the retry also fails, GLIDE attaches the original rejection as `cause` when possible; node-redis surfaces the retry rejection unmodified because disconnect failures may be shared across callers. A healed retry is indistinguishable from first-attempt success in DialCache metrics. Monitor server-side `INFO commandstats` for unexpected `EVAL` volume or rejected `EVALSHA` calls. -A tracked write rejected by an active future watermark uses `UNLINK` to remove the value key — the placeholder it just stored, along with any logically stale frame — without synchronously freeing it on Redis's command path. The mutation protocol therefore requires a server that implements `UNLINK` (Redis 4.0 or later, or a compatible Valkey release). Command-restricted Redis ACLs must allow the client to issue the native `GET`, `MGET`, and `SET` commands — `SET` newly carries every write, where the previous protocol wrote only through scripts — plus `EVALSHA` (the steady-state dispatch for both mutation scripts) and `EVAL` (both adapters recover a flushed script cache by re-sending script sources, never via `SCRIPT LOAD`). Server versions differ on whether script-invoked commands are also checked against the invoking user, so grant what the mutation scripts invoke as well: `TIME`, `GET`, `SET`, and `PTTL` (both scripts), plus the stamp's `PEXPIRE`, `UNLINK`, `GETRANGE`, and `SETRANGE`. Verify those grants before upgrading, because the failure amplitude of a persistent stamp fault changed. A sustained stamp failure (denied command, a proxy rejecting `EVALSHA`) still lands every paired `SET`, so each tracked write replaces the last served value with an unreadable placeholder while also suppressing process-local publication — within one TTL horizon the source absorbs full traffic, where the previous protocol degraded to serving stale values until expiry. DialCache's integration matrix covers Redis 6.2 and Valkey 8. +Command-restricted Redis ACLs must allow native `GET`, `MGET`, and `SET`, plus `EVALSHA` and `EVAL` for invalidation recovery. If script-invoked commands are checked separately, the invalidation script needs `GET`, `SET`, and `PTTL`. Redis `TIME`, `MULTI`, `EXEC`, `WATCH`, `UNLINK`, and `SCRIPT LOAD` are not used by DialCache. The integration matrix covers Redis 6.2 and Valkey 8. #### Remote read deadlines and async liveness @@ -428,20 +428,22 @@ Writes, invalidations, async `cacheConfigProvider` calls, and custom serializer #### Serialization -The core Redis boundary is the client-agnostic `DialCacheRedisClient` interface. Writes accept serialized values as `string | Buffer`; reads return a `DecodedRedisFrame` — the decoded `string | Buffer` payload plus the frame header's `createdAtMs` (Redis server time for tracked frames, the writer's informational client clock for untracked ones) — and the interface does not expose client commands or wire encodings. The shared `encodeRedisFrame`, `encodeTrackedRedisPlaceholder`, `decodeRedisFrame`, and `decodeTrackedRedisFrame` helpers, the `resolveTrackedRedisWriteReply`, `validateRedisSetReply`, and `validateRedisScriptInvalidationReply` reply helpers, the `ceilSupportedCacheTtlMs` TTL guard, and the tracked stamp and invalidation Lua sources are available from `dialcache/redis-protocol`, so custom adapters can reuse the bundled adapters' exact framing, miss, watermark-fencing, TTL-domain, and reply rules. A custom tracked write must pass the stamp script `KEYS = [valueKey, watermarkKey]` and `ARGV = [cacheTtlMs, nonce]`, running `cacheTtlMs` through `ceilSupportedCacheTtlMs` and using the result for both the paired `SET`'s `PX` and `ARGV[1]` (the stamp script re-validates the same domain server-side as defense in depth), with the nonce from the same `encodeTrackedRedisPlaceholder` call; `resolveTrackedRedisWriteReply` maps the reply, failing the write with the root-exported `DialCacheRedisPlaceholderLostError` when the stamp replies `2`. Custom adapters can throw the root-exported `DialCacheRedisPayloadError`, `DialCacheRedisPayloadEncodingError`, `DialCacheRedisProtocolError`, and `DialCacheRedisPlaceholderLostError` classes to distinguish malformed replies, unsupported encodings, reply-domain violations, and lost placeholders in logs. DialCache records bounded `cache_read`, `cache_write`, or `invalidation` metrics by failure site. +The core Redis boundary is the client-agnostic `DialCacheRedisClient` interface. Writes accept serialized values as `string | Buffer`; reads return a `DecodedRedisFrame` — the decoded `string | Buffer` payload plus the frame header's writer-client `createdAtMs` — and the interface does not expose client commands or wire encodings. That timestamp is correctness-relevant: custom clients must return the frame's valid nonnegative safe-integer epoch timestamp rather than a constant. `RedisWriteRequest` contains only `valueKey`, `cacheTtlMs`, and `value`, and `write()` returns `void`; watermark ownership is exclusive to tracked reads and invalidation. + +The shared `encodeRedisFrame`, `decodeRedisFrame`, and `decodeTrackedRedisFrame` helpers, the `validateRedisSetReply` and `validateRedisScriptInvalidationReply` reply helpers, the `ceilSupportedCacheTtlMs` TTL guard, and the invalidation Lua source are available from `dialcache/redis-protocol`. A custom write samples `Date.now()` once, calls `encodeRedisFrame(value, createdAtMs)`, and sends one `SET valueKey frame PX cacheTtlMs` after validating the TTL. A custom tracked read atomically obtains `[value, watermark]` from the primary and passes both replies to `decodeTrackedRedisFrame`; a missing watermark is treated as zero. Invalidation passes `KEYS = [watermarkKey]` and `ARGV = [futureBufferMs, invalidatedAtMs]`, with one stable client timestamp reused across retries. Custom adapters can throw the root-exported `DialCacheRedisPayloadError`, `DialCacheRedisPayloadEncodingError`, and `DialCacheRedisProtocolError` classes to distinguish malformed replies, unsupported encodings, and reply-domain violations. DialCache records bounded `cache_read`, `cache_write`, or `invalidation` metrics by failure site. Redis values use a compact binary frame: ```text -byte 1 format version: 1 = servable, 0 = unreadable tracked placeholder -bytes 2-9 uint64 big-endian: Redis server time for a promoted tracked frame, - informational client time for an untracked frame, or the random - per-write nonce while a tracked placeholder awaits its stamp +byte 1 format version: 1 +bytes 2-9 uint64 big-endian: writer application time in epoch milliseconds byte 10 payload encoding (0 = UTF-8, 1 = raw binary) bytes 11... serialized payload (optionally zstd-compressed; see Compression) ``` -Adapters build frames in the Node process. Untracked frames come from `encodeRedisFrame` and carry an informational client-clock timestamp that untracked reads never consult. Tracked frames start as `encodeTrackedRedisPlaceholder` output — version byte `0`, with a random per-write nonce in the timestamp bytes — which no read path serves; the stamp script verifies the nonce and promotes the frame to version `1` with Redis server time using Lua's `struct` library, and adapters decode it with Node's buffer primitives. Redis TTL is authoritative, so expiry metadata is not duplicated in the frame. `payload` is produced by the operation's serializer, or by `JsonSerializer` by default. Custom serializers can return either `string` or `Buffer`. Payloads stored raw keep their exact serialized bytes: strings are stored as UTF-8 and Buffers byte-for-byte without base64 expansion, except that binary output beginning with a [compression envelope byte](#compression) (`0x00`–`0x02`) gains a one-byte escape prefix on the wire. Payloads at or above the compression threshold may instead be stored as a zstd envelope (see [Compression](#compression)), so wire bytes for large values are not the serializer's output. Adapters return the frame payload as-is; the envelope — including restoring a compressed string's representation before `serializer.load` — is interpreted by the core above them. +Adapters build complete frames in the Node process with `encodeRedisFrame` and decode them with Node buffer primitives. The version-1 value-envelope format, Redis value-key derivation, and decimal watermark encoding are unchanged. That wire compatibility does not make old tracked state safe to carry across the protocol cutover: old watermark lifetimes were derived for the old write protocol. + +Redis physical TTL remains authoritative for normal expiry; the frame timestamp supports future-frame rejection and shadow value-age observability. `payload` is produced by the operation's serializer, or by `JsonSerializer` by default. Custom serializers can return either `string` or `Buffer`. Payloads stored raw keep their exact serialized bytes: strings are stored as UTF-8 and Buffers byte-for-byte without base64 expansion, except that binary output beginning with a [compression envelope byte](#compression) (`0x00`–`0x02`) gains a one-byte escape prefix on the wire. Payloads at or above the compression threshold may instead be stored as a zstd envelope (see [Compression](#compression)), so wire bytes for large values are not the serializer's output. Adapters return the frame payload as-is; the envelope — including restoring a compressed string's representation before `serializer.load` — is interpreted by the core above them. DialCache uses native `JSON.stringify` and `JSON.parse` by default. There is no runtime validation pass, so the default adds no traversal beyond JSON serialization itself. A top-level `undefined` result is supported with an internal sentinel. @@ -475,6 +477,12 @@ The compile-time guard rejects known incompatible shapes such as `Date`, `Map`, This guard is deliberately conservative and is not a proof of runtime data. TypeScript cannot detect non-finite numbers, cyclic/shared references, runtime getter or `toJSON` behavior, or data-only class instances that look like plain objects. Opaque, generic, or deeply recursive types may also require an explicit serializer. Providing `Serializer` (including an explicitly typed `JsonSerializer`) is a trusted caller assertion; DialCache does not serialize-and-deserialize again to validate it. +#### Protocol cutover + +The old and new tracked-write protocols must not coexist. Before enabling this release for a namespace, stop and drain every old writer and invalidator plus in-flight fallbacks, shadow work, Redis client queues, and other operations that can still write tracked state. Then purge every tracked value — complete or placeholder — and every watermark in the affected namespace. A full namespace flush is the simplest option when the Redis deployment is dedicated; untracked complete values may be retained. + +Alternatively, keep all traffic disabled until every old tracked value and watermark has expired naturally. That is safe only when the maximum remaining lifetime of both classes is bounded, no watermark is persistent, and the wait covers the old release's maximum value TTL and future-buffer-derived watermark TTL. This library intentionally does not implement the external deployment gate. + #### Compression DialCache transparently compresses serialized Redis payloads with zstd (level 3, via `node:zlib`) when they are at least 4096 serialized bytes, and stores the compressed form only when it is smaller than the raw stored form. Compression sits below the serializer and above the Redis client, so serializers, adapters, and the frame layout are unaffected. The first byte of a binary frame payload written by a release with payload compression is an envelope byte: `0x01` marks a compressed UTF-8 string and `0x02` compressed binary output, each followed by the zstd frame, while `0x00` is an escape prefix for raw binary serializer output whose own first byte is `0x00`–`0x02` (readers strip the prefix and never decompress it; the escape applies even with `compression: false`). Payloads below the threshold are otherwise stored byte-identical to earlier DialCache releases; only binary output beginning with an envelope byte gains the one-byte escape. @@ -501,7 +509,7 @@ Each write records a bounded compression outcome (`compressed`, `below_threshold #### Shadow validation -Shadow mode runs a sampled, detached Redis path for tracked or untracked keys without allowing that path to serve the caller. A Redis hit is compared with the source of truth (SoT); a clean Redis miss can be filled from the caller-accepted SoT value. Redis serving and shadow execution are independent, and shadowing is opt-in per use case through `shadow.ramp`: +Shadow mode runs a sampled, detached Redis path for tracked or untracked keys without allowing that path to serve the caller. A Redis hit is compared with the source of truth (SoT); a semantic Redis miss can be filled from the caller-accepted SoT value. Redis serving and shadow execution are independent, and shadowing is opt-in per use case through `shadow.ramp`: ```ts import { CacheLayer, DialCache, DialCacheKeyConfig } from "dialcache"; @@ -521,7 +529,7 @@ const getUser = dialcache.cached( keyType: "user_id", useCase: "GetUser", cacheKey: (userId) => userId, - // Optional for shadowing; adds watermark fencing to Redis reads and fills. + // Optional: make Redis reads watermark-aware; writes remain complete-frame SETs. trackForInvalidation: true, // Optional: override strict deep equality with use-case semantics. shadowComparator: (cached, source) => @@ -541,10 +549,10 @@ const getUser = dialcache.cached( ); ``` -Shadow work is eligible only when a valid remote TTL/policy exists, its effective `shadow.ramp` selects the exact cache key, a configured metrics adapter implements `shadowValidation`, and capacity is available. Tracked and untracked Redis keys are both eligible; each keeps its existing read and write mode. Logging is supplemental to the metric; enabling `logMismatches` does not activate shadow work without the metrics hook. The bundled Prometheus and Datadog adapters implement that hook. There are two paths: +Shadow work is eligible only when a valid remote TTL/policy exists, its effective `shadow.ramp` selects the exact cache key, a configured metrics adapter implements `shadowValidation`, and capacity is available. Tracked and untracked Redis keys are both eligible; each keeps its existing read mode and uses the common complete-frame write path. Logging is supplemental to the metric; enabling `logMismatches` does not activate shadow work without the metrics hook. The bundled Prometheus and Datadog adapters implement that hook. There are two paths: - When remote serving is enabled and produces a Redis hit, DialCache retains the exact serialized payload that supplied the caller as `C0`. -- When the remote policy is valid but disabled specifically by `ramped_down`, DialCache starts a detached Redis read for `C0` using the key's existing tracked or untracked mode. Its result can be validated or used to decide whether a clean miss may be filled, but can never supply the caller or populate an in-memory layer. +- When the remote policy is valid but disabled specifically by `ramped_down`, DialCache starts a detached Redis read for `C0` using the key's existing tracked or untracked mode. Its result can be validated or used to decide whether a semantic miss may be filled, but can never supply the caller or populate an in-memory layer. A missing or invalid remote policy, config-provider failure, absent Redis client, disabled call, omitted metrics hook, zero/omitted shadow ramp, cohort exclusion, capacity rejection, or earlier request-local/process-local hit does not launch a shadow-only Redis path. Shadow work begins only if normal traversal reaches the Redis layer. A normally enabled remote miss already follows the caller's ordinary fallback-and-fill path and does not launch a duplicate shadow fill. @@ -553,24 +561,24 @@ On a served hit, DialCache returns the already-decoded cached value before start The detached job uses this bounded algorithm: 1. Obtain the original Redis payload as `C0` using the key's existing tracked or untracked read mode. -2. If `C0` is missing, wait for the caller's successfully accepted `S` after its configured fallback boundary and attempt one normal Redis write in the same mode using the resolved TTL. Before the whole-job deadline, emit `filled` when Redis accepts it, `fill_blocked` when a tracked invalidation watermark rejects it, or `fill_error` when serialization or the write fails. `fill_blocked` is not produced by compliant untracked writes. +2. If `C0` is missing, wait for the caller's successfully accepted `S` after its configured fallback boundary and attempt one normal Redis write using the resolved TTL. Before the whole-job deadline, emit `filled` when Redis accepts it or `fill_error` when serialization or the write fails. A tracked fill can be physically stored yet remain fenced on later tracked reads. 3. If `C0` is non-null, obtain `S`, deserialize an isolated snapshot of `C0`, and run the default or custom semantic comparator. Any non-null `C0` is observation-only: DialCache never repairs or overwrites it, including when deserialization fails. 4. If `C0` and `S` match semantically, emit `match` without another Redis read. 5. Otherwise, reread Redis directly in the same mode as `C1`, bypassing request-local and process-local cache. -6. If `C1` is missing or differs byte-for-byte from `C0`, emit `superseded`; if it is identical, emit `mismatch`. +6. If `C1` is missing under the normal value/watermark protocol or differs byte-for-byte from `C0`, emit `superseded`; if it is identical, emit `mismatch`. A future-dated tracked `C1` records its offset but remains available for this payload comparison, so a reader-clock step does not change the verdict. 7. If the confirmation read fails or reaches its Redis-read deadline, emit `confirmation_error`. -Here a clean miss means the semantic Redis read returned `null`; it does not include a non-null payload that later fails deserialization. A caller fallback rejection or timeout never becomes accepted `S` and never starts the fill. +Here a semantic miss means the Redis read returned `null`; it does not include a non-null payload that later fails deserialization. A physical tracked frame rejected by its watermark, timestamp domain, or future-time check is therefore a miss and may be replaced by the fill. A caller fallback rejection or timeout never becomes accepted `S` and never starts the fill. -Both detached Redis reads use the effective `remoteReadTimeoutMs` and the key's normal protocol. Every clean-miss fill uses the same serializer, TTL, and timestamp semantics as an ordinary fill — server time for tracked fills, informational client time for untracked ones. A tracked fill blanks the key with its placeholder before publishing, so a lost or raced stamp can leave a previously readable value unreadable until the value TTL, and `fill_error` includes that benign lost-placeholder outcome. Tracked `C0` and `C1` reads remain watermark-aware and are routed to primaries by the bundled adapters, while tracked fills also retain the ordinary invalidation watermark. Untracked reads use the ordinary one-key read route, which has no shadow-specific primary guarantee, and untracked fills use the ordinary TTL write without a watermark. Strings compare exactly, Buffers compare by bytes, and string/Buffer pairs compare by their UTF-8 bytes. DialCache does not deserialize `C1`, compare it with `S`, or chase another version. +Both detached Redis reads use the effective `remoteReadTimeoutMs` and the key's normal protocol. Every semantic-miss fill uses the same serializer, TTL, complete-frame SET, and client-clock timestamp semantics as an ordinary fill. Tracked `C0` and `C1` reads remain watermark-aware and are routed to primaries by the bundled adapters. The fill itself never reads or mutates the watermark. Untracked reads use the ordinary one-key read route, which has no shadow-specific primary guarantee. Strings compare exactly, Buffers compare by bytes, and string/Buffer pairs compare by their UTF-8 bytes. DialCache does not deserialize `C1`, compare it with `S`, or chase another version. The detached scheduler, Redis-read deadline timers, and overall shadow deadline timer are unreferenced, so they do not keep an otherwise idle process alive. Detachment is asynchronous work on the Node event loop, not a worker thread: synchronous source, serializer, or comparator work can still occupy the event loop after the request path has been released. Detached execution retains the original `cached()` argument references or `getOrLoad()` loader closure; DialCache cannot generically clone them. Treat object arguments, captured source-selection state, and the returned `S` as immutable, or snapshot them before invoking DialCache. Mutating them after the caller continues can compare or serialize a value that no longer corresponds to the already-built key. -`shadowMaxInFlight` is a per-instance positive safe integer and defaults to `1`. It counts admitted jobs until shadow-owned Redis/source/serializer/comparator work settles, including detached reads or dispatched writes whose DialCache deadline already elapsed. The optional `C1` and clean-miss fill remain in the original slot. On a ramped-down path, shadow work shares the caller's SoT promise; once the shadow deadline expires, the raw caller-owned loader may continue without retaining the shadow slot, including when `fallbackTimeoutMs` is `null`. DialCache also suppresses another job for the same exact key while shadow-owned work remains active. There is no queue: exact-key duplicates and work above the instance cap are dropped and reported as `dropped`. Separate instances have independent limits, so this is not a fleet-wide source-of-truth or Redis concurrency cap. +`shadowMaxInFlight` is a per-instance positive safe integer and defaults to `1`. It counts admitted jobs until shadow-owned Redis/source/serializer/comparator work settles, including detached reads or dispatched writes whose DialCache deadline already elapsed. The optional `C1` and semantic-miss fill remain in the original slot. On a ramped-down path, shadow work shares the caller's SoT promise; once the shadow deadline expires, the raw caller-owned loader may continue without retaining the shadow slot, including when `fallbackTimeoutMs` is `null`. DialCache also suppresses another job for the same exact key while shadow-owned work remains active. There is no queue: exact-key duplicates and work above the instance cap are dropped and reported as `dropped`. Separate instances have independent limits, so this is not a fleet-wide source-of-truth or Redis concurrency cap. -Each job has one monotonic deadline across detached `C0`, the SoT result, serializer work, comparison, optional `C1`, and clean-miss fill. Served-hit timing begins when its detached validation callback starts. On a ramped-down path, timing begins immediately before the caller's SoT invocation so synchronous source work that runs before admission still consumes the same budget. Each Redis read also has its effective read deadline. A finite `fallbackTimeoutMs` is reused as the overall shadow budget. When `fallbackTimeoutMs` is `null`, the normal fallback remains intentionally unbounded, but detached shadow work still uses a 60-second budget. Once timeout delivery marks a job abandoned, DialCache releases retained `C0` references and prevents later phases from starting. +Each job has one monotonic deadline across detached `C0`, the SoT result, serializer work, comparison, optional `C1`, and semantic-miss fill. Served-hit timing begins when its detached validation callback starts. On a ramped-down path, timing begins immediately before the caller's SoT invocation so synchronous source work that runs before admission still consumes the same budget. Each Redis read also has its effective read deadline. A finite `fallbackTimeoutMs` is reused as the overall shadow budget. When `fallbackTimeoutMs` is `null`, the normal fallback remains intentionally unbounded, but detached shadow work still uses a 60-second budget. Once timeout delivery marks a job abandoned, DialCache releases retained `C0` references and prevents later phases from starting. JavaScript promises and Redis writes do not provide a general cancellation or transaction boundary. Work already dispatched may continue and keeps the shadow slot until it settles. A write rejection, `fill_error`, or shadow `timeout` after dispatch does not prove that Redis was unchanged; the command may have executed before its result became unavailable. Conversely, `filled` means the semantic client returned success before the shadow deadline, not that the value is still present. Give dependencies finite native budgets and treat shadow outcomes as best-effort operational evidence. @@ -582,11 +590,11 @@ A `match` means application-level value equality: DialCache retains the semantic frame returned by the Redis client — its `string | Buffer` payload and `createdAtMs` — but never exposes it to the comparator. After the source read completes, detached work calls the same effective serializer's `load` method to create an independent cached snapshot, then compares that snapshot with the raw value returned by the source loader. It does not reuse the cached object already returned to a served-hit caller, so caller mutation cannot contaminate validation. No payload copy, shadow deserialization, deep comparison, or hash is added to the served-hit request path. -The effective serializer's `load` method therefore runs a second time for a sampled served hit and once in detached work for a shadow-only hit. It must be repeatable, non-mutating, and return independently usable values. On a clean miss, its `dump` method may run after the caller has received `S`, so `S` must remain immutable through detached serialization. A custom `DialCacheRedisClient` must return an operation-owned frame whose string/Buffer payload contents remain stable after `read()` settles. Comparing the deserialized cached snapshot with the raw source value intentionally detects lossy serialization; use a custom comparator only when such normalization or ignored fields are valid use-case semantics. +The effective serializer's `load` method therefore runs a second time for a sampled served hit and once in detached work for a shadow-only hit. It must be repeatable, non-mutating, and return independently usable values. On a semantic miss, its `dump` method may run after the caller has received `S`, so `S` must remain immutable through detached serialization. A custom `DialCacheRedisClient` must return an operation-owned frame whose string/Buffer payload contents remain stable after `read()` settles. Comparing the deserialized cached snapshot with the raw source value intentionally detects lossy serialization; use a custom comparator only when such normalization or ignored fields are valid use-case semantics. -Shadow metrics use the bounded outcomes `match`, `mismatch`, `superseded`, `filled`, `fill_blocked`, `fill_error`, `redis_error`, `source_error`, `deserialization_error`, `comparison_error`, `confirmation_error`, `timeout`, and `dropped`. `redis_error` applies to the initial shadow-only `C0` read; `confirmation_error` applies to `C1`; `fill_blocked` applies only when a tracked watermark rejects the write. A clean `C0` miss is an ordinary `miss{layer="remote_shadow"}` and terminates with a fill, source, or timeout outcome rather than a second shadow outcome for the miss itself. Labels never contain cache ids, values, payloads, Redis keys, or raw exception text. +Shadow metrics use the bounded outcomes `match`, `mismatch`, `superseded`, `filled`, `fill_error`, `redis_error`, `source_error`, `deserialization_error`, `comparison_error`, `confirmation_error`, `timeout`, and `dropped`. `redis_error` applies to the initial shadow-only `C0` read and `confirmation_error` applies to `C1`. A semantic `C0` miss is an ordinary `miss{layer="remote_shadow"}` and terminates with a fill, source, or timeout outcome rather than a second shadow outcome for the miss itself. Labels never contain cache ids, values, payloads, Redis keys, or raw exception text. -A `match` or `mismatch` verdict additionally records the validated value's age through the optional `observeShadowValueAge` adapter hook: the observing process's epoch clock minus the validated frame's `createdAtMs` (the served `C0` frame, or the detached `C0` frame on a ramped-down path), in seconds, clamped at zero. The age is captured at verdict time, so a mismatch age lands one confirmation read after the comparison itself. A confirmed `mismatch` age therefore measures how long the stale value had been readable when validation caught it. Tracked frames are stamped with Redis server time and untracked frames with the writer's client clock, so the age mixes clocks and is coarse operational evidence rather than a precise measurement. Outcomes that deliver no verdict on a retained value — including `superseded`, `filled`, and every error or timeout outcome — record no age. The hook does not gate shadow eligibility; only `shadowValidation` does. +A `match` or `mismatch` verdict additionally records the validated value's age through the optional `observeShadowValueAge` adapter hook: the observing process's epoch clock minus the writer-stamped frame's `createdAtMs` (the served `C0` frame, or the detached `C0` frame on a ramped-down path), in seconds, clamped at zero. The age is captured at verdict time, so a mismatch age lands one confirmation read after the comparison itself. A confirmed `mismatch` age therefore measures how long the stale value had been readable when validation caught it. Because writer and observer are different application processes, their clock offset makes the age coarse operational evidence rather than a precise measurement. Outcomes that deliver no verdict on a retained value — including `superseded`, `filled`, and every error or timeout outcome — record no age. The hook does not gate shadow eligibility; only `shadowValidation` does. Confirmed-mismatch logging is separately opt-in through `shadow.logMismatches`; it never replaces the required `shadowValidation` metric or emits for a mismatch candidate that becomes `superseded`. The single warning contains `cacheNamespace`, `useCase`, `keyType`, `outcome: "mismatch"`, `cacheKey`, `cachedValueJson`, and `sourceValueJson`. `cacheKey` is the logical DialCache URN capped at 2 KiB, not the physical Redis storage key. DialCache independently applies native `JSON.stringify` to the deserialized cached snapshot and raw source value supplied to the comparator, then caps each resulting string at 8 KiB. A byte-clipped field ends in `...[truncated]`, counted inside its cap. If native JSON throws or returns `undefined`, the corresponding JSON field is `null`; the other side is still attempted. DialCache does not compute a textual diff or call the configured serializer again for logging. @@ -598,11 +606,11 @@ Detached Redis reads, serializer loads/dumps, payload sizes, and read/write erro The command amplification is bounded: a selected served hit adds one SoT read and adds `C1` only for a semantic mismatch candidate; a selected ramped-down hit adds detached `C0`, reuses the caller's existing SoT read, and likewise adds `C1` only for a candidate; a selected ramped-down miss adds detached `C0` and at most one fill in the key's existing mode. `superseded` means only that the original observation could not be confirmed. `mismatch` means the exact `C0` payload survived another Redis read after the SoT disagreement; it is not a cross-system atomic snapshot or a guarantee that the mismatch persists. For an untracked key it is also not proof of primary freshness or invalidation safety. -The initial `C0` read and later fill are not atomic. The fill is a normal overwrite, not a compare-and-set or write-if-still-missing operation: another writer can populate Redis after the clean miss and be overwritten by the shadow fill. Tracked invalidation watermarks still fence tracked writes using Redis time, so size `futureBufferMs` to cover the complete SoT, serialization, client queue, network, and write interval when stale-publication protection matters. An untracked shadow fill has no such fence and retains the ordinary TTL-based last-writer-wins contract; because it is detached, an older accepted source value may be written after a concurrent source mutation and remain until expiry. Shadow mode never repairs a non-null `C0`, refreshes its TTL, invalidates, evicts local state, or changes the value returned to the caller. +The initial `C0` read and later fill are not atomic. The fill is a normal overwrite, not a compare-and-set or write-if-still-missing operation: another writer can populate Redis after the semantic miss and be overwritten by the shadow fill. For tracked keys, the next atomic value-and-watermark read still rejects a frame whose client timestamp is at or before the watermark, so size `futureBufferMs` to cover the complete SoT, serialization, client queue, network, write interval, and fleet clock-skew budget when stale-serving protection matters. An untracked shadow fill has no such read fence and retains the ordinary TTL-based last-writer-wins contract; because it is detached, an older accepted source value may be written after a concurrent source mutation and remain until expiry. Shadow mode never repairs a frame returned as a non-null semantic `C0`; a physical frame rejected by normal tracked-read semantics is a miss and may be overwritten with a fresh TTL. Shadow work never invalidates, evicts local state, or changes the value returned to the caller. A served-hit sample invokes the wrapped function or inline loader as an additional source read, so that loader must be safe to call for observation. A ramped-down sample reuses the caller's ordinary invocation and does not add another SoT call. -For valid policies, shadow-specific source calls, cache-path Redis traffic, returned values, and metrics are unchanged when `shadow` is omitted or `shadow.ramp` is `0`. Shadow policy is grouped under `DialCacheKeyConfig.shadow`; consumers of the former flat ramp field must migrate to `shadow: { ramp }`. The public constructor, static defaults, and runtime provider results reject the removed field. `DialCacheKeyConfig.disabled()` explicitly disables the shadow ramp and mismatch logging. `shadowComparator` remains a typed `cached()` / `getOrLoad()` option because it defines stable use-case equality, while `shadowMaxInFlight` remains a per-instance concurrency limit. The clean-miss bootstrap adds no additional ramp knob, Redis protocol operation, metric instrument, or label key; enabling shadowing authorizes the same-mode Redis write described above. Untracked keys now participate when they have a nonzero effective shadow ramp and an observable metrics hook, so deployments that previously supplied such a ramp while relying on the tracked-only eligibility rule must set it to `0` before upgrading if they do not want the added SoT reads, Redis traffic, possible fills, and opted-in mismatch logs. Exported unions include `remote_shadow` in `MetricLayer` and `superseded`, `filled`, `fill_blocked`, `fill_error`, `redis_error`, and `confirmation_error` in `ShadowValidationOutcome`. TypeScript consumers with exhaustive switches or `Record` values must include those cases, and dashboards restricted to `layer="remote"` intentionally exclude detached traffic. +For valid policies, shadow-specific source calls, cache-path Redis traffic, returned values, and metrics are unchanged when `shadow` is omitted or `shadow.ramp` is `0`. Shadow policy is grouped under `DialCacheKeyConfig.shadow`; consumers of the former flat ramp field must migrate to `shadow: { ramp }`. The public constructor, static defaults, and runtime provider results reject the removed field. `DialCacheKeyConfig.disabled()` explicitly disables the shadow ramp and mismatch logging. `shadowComparator` remains a typed `cached()` / `getOrLoad()` option because it defines stable use-case equality, while `shadowMaxInFlight` remains a per-instance concurrency limit. The semantic-miss bootstrap adds no additional ramp knob, Redis protocol operation, metric instrument, or label key; enabling shadowing authorizes the same Redis write described above. Untracked keys participate when they have a nonzero effective shadow ramp and an observable metrics hook. Exported unions include `remote_shadow` in `MetricLayer` and `superseded`, `filled`, `fill_error`, `redis_error`, and `confirmation_error` in `ShadowValidationOutcome`. TypeScript consumers with exhaustive switches or `Record` values must include those cases, and dashboards restricted to `layer="remote"` intentionally exclude detached traffic. ## Cached-value ownership @@ -659,19 +667,23 @@ Invalidation writes a Redis watermark at `{encodedNamespace:encodedKeyType:encod The internal `:dialcache-frame-v1` suffix identifies values written with DialCache's binary protocol. Watermarks are stored as decimal timestamps. -A cached Redis value whose Redis-created timestamp is older than or equal to the watermark is treated as stale and refreshed through fallback. `invalidateRemote(keyType, id, futureBufferMs)` sets the watermark to the greater of its existing value and Redis's current time plus the buffer. While that future window is active, an invocation that reaches the tracked Redis read treats the covered value as a miss. Native `MGET` must transfer an existing stale frame before the Node decoder can reject it, so completed reads can repeatedly pay the full stale-payload transfer during a nonzero buffer window. If a successful fallback then reaches the tracked Redis write while the watermark still fences it, the stamp script reports the write as blocked, unlinks the value key — the placeholder that write just stored, along with the logically stale frame it replaced — and DialCache suppresses the corresponding process-local population; later reads of that entry avoid retransferring its payload. The fallback value still returns to its caller. A read failure or timeout never reaches that write-side cleanup, so a large stale value can continue to consume network bandwidth and trigger `cache_read_timeout` until another completed read cleans it up or its TTL expires. Request-local memoization remains unconditional. A ramped-out invocation without shadow work does not consult the watermark; a selected shadow path for that tracked key does consult it for `C0`, `C1` when needed, and any clean-miss fill, although caller-path request-local/process-local publication remains independent. +A cached Redis value whose writer-provided `createdAtMs` is older than or equal to the watermark is a tracked miss. `invalidateRemote(keyType, id, futureBufferMs)` proposes the invalidating process's `Date.now()` plus the buffer, and Lua keeps the greater of that proposal and the existing watermark. A tracked read obtains value and watermark in one primary-routed `MGET`; a missing watermark is the zero baseline, malformed or out-of-range decimal state fails closed, and `createdAtMs <= watermark` misses. Redis also returns `nil` for a wrong-type member of `MGET`, so a wrong-type watermark has the same zero-baseline behavior as an absent one until the next explicit invalidation repairs it. Native `MGET` must transfer the full stored frame before Node can apply the fence verdict, so a future window can repeatedly transfer a large fenced payload. Fallback still returns normally and writes one complete replacement frame even when that frame remains behind the watermark. A tracked invocation that reaches the Redis read/write path does not publish its fallback directly to process-local cache; a later validated Redis hit may warm it. Local-only, remote-policy-disabled, and ramped-down paths remain governed by local policy, while request-local memoization remains unconditional. A ramped-out invocation without shadow work does not consult Redis; selected tracked shadow reads remain watermark-aware. + +All serving timestamps come from application-process epoch clocks; DialCache does not call Redis `TIME`, estimate an offset, or compensate for skew. Participating application nodes therefore need external clock synchronization and monitoring. Healthy managed node pools commonly stay close, but Kubernetes does not guarantee a maximum offset, and pauses or NTP faults can be much larger than normal millisecond-scale skew. A tracked writer ahead of a reader produces fail-closed misses until the reader clock catches up; untracked reads do not gate serving on the informational stamp. Operation durations and deadlines continue to use the monotonic `performance.now()` clock. -The bundled timestamp protocol assumes that system clocks are synchronized across every Redis node eligible for primary promotion. Redis does not guarantee that `TIME` is monotonic across nodes, and DialCache does not detect or compensate for cross-node clock skew. If this deployment assumption is violated, failover can temporarily suppress tracked cache fills or allow a pre-invalidation value to remain readable until it expires or a later invalidation advances the watermark past its timestamp. +Watermarks are invalidation state, not disposable cache entries. The Redis deployment must preserve them for their derived TTL: use `noeviction` or an equivalent guarantee for deployments that rely on the read-time fence, and choose persistence and failover guarantees appropriate to the application's consistency requirements. If a watermark is lost, a tracked read treats it as zero and may serve an existing frame that the lost watermark had fenced. Alert on memory headroom and rejected writes under `noeviction`; if another eviction policy is used, also alert on `evicted_keys`. Redis replication is asynchronous by default, and DialCache does not issue `WAIT` or provide strong consistency across failover. -Watermarks are invalidation state, not disposable cache entries. The Redis deployment must preserve them for their derived TTL: use `noeviction` or an equivalent guarantee for deployments that rely on the publication fence, and choose persistence and failover guarantees appropriate to the application's consistency requirements. A missing watermark makes tracked reads miss, but a later tracked write cannot distinguish an empty cache from lost invalidation history; it creates a new baseline watermark and can publish fallback data that the lost future watermark would have rejected. Redis replication is asynchronous by default, and DialCache does not issue `WAIT` or provide strong consistency across failover. +Tracked Redis value TTLs are capped at one hour; a dispatched write configured above the cap records `tracked_ttl_clamped`. Only invalidation creates or updates a watermark. Its TTL is `max(existing TTL, 2 hours, watermark - invalidatedAtMs + 1 hour + 1 minute)`; an existing persistent watermark stays persistent. Reads and writes never extend it. Under the documented clock-skew and in-flight-work bounds, this makes every finite watermark outlive every value it can fence, including a maximum future-buffer proposal. Raising the tracked-value cap or shrinking the watermark floor is a protocol transition that requires another no-overlap gate, drain, and purge; changing both constants in one new binary cannot lengthen watermarks already written by an older invalidator. -Tracked writes create a baseline watermark and extend its TTL to at least the value TTL plus one minute. Neither tracked writes nor invalidation shorten a longer or persistent watermark TTL; invalidation extends it to at least the remaining future-buffer window plus one minute. There is no fixed watermark retention floor, and reads do not extend watermark lifetime. +`futureBufferMs` must be a nonnegative safe integer no greater than 31,536,000,000 (a fixed 365-day duration). The default is zero, but zero provides no stale-serving protection once a writer timestamp advances past the watermark. Every production invalidation should pass a named, application-owned nonzero value based on that application's measured or conservatively bounded timings; there is no universally safe library value. -`futureBufferMs` must be a nonnegative safe integer no greater than 31,536,000,000 (a fixed 365-day duration). The default is zero, but zero provides no stale-publication protection once Redis time advances. Every production invalidation should pass a named, application-owned nonzero value based on that application's measured or conservatively bounded timings; there is no universally safe library value. +Let `Dmax` be the maximum elapsed time from invalidation sampling until a stale pre-mutation `SET` can become visible in Redis, `S` the maximum writer-clock lead over the invalidator across participating application nodes, and `M` an operational margin. Callers that require stale-serving protection must satisfy `futureBufferMs >= Dmax + S + M`. Bound `Dmax` through source visibility or replication lag, the remaining tail of any fallback that may observe the pre-mutation value, `serializer.dump`, Redis client queueing or reconnect/offline-queue delay, network transit, and Redis execution and visibility. An unbounded offline queue or retry path makes a finite `Dmax` impossible. Invalidate only after the source mutation commits. The dangerous skew direction is a fast writer and slow invalidator: an undersized buffer can let delayed stale work receive a timestamp above the watermark and remain readable until expiry or another invalidation. The reverse direction is conservative. Overestimating the buffer increases fallback load, full-payload `MGET` transfer, and full-frame `SET`, replication, and AOF churn on hot large-value keys. It does not delay or suppress returning fallback values to callers. -Size the buffer to cover the maximum expected negative clock skew between promotion-eligible Redis nodes plus the complete interval in which stale data could still reach the Redis write: source visibility or replication lag, the full remaining tail of any fallback that may already have observed the pre-mutation value, `serializer.dump`, Redis client queue and network latency, the placeholder write and the stamp script that assigns its server timestamp, and a safety margin. Invalidate only after the source mutation commits. Underestimating this interval can allow a delayed stale fallback to repopulate Redis after the watermark window ends. Overestimating it lengthens the tracked Redis miss/write-suppression window described above, increasing fallback load and, until write-side cleanup succeeds, stale-payload transfer and read-timeout risk without publishing stale values. Each fenced write inside the window also stores its full placeholder payload before the stamp unlinks it, so a long buffer on a hot large-value key adds allocator, replication, and AOF churn the previous fence-before-store protocol never paid. A larger buffer does not delay or suppress returning fallback values to callers. +The fixed one-minute watermark-TTL margin is retention slack after the covered visibility bound; it is not a substitute for `Dmax`. The operational contract should include the full post-sample dispatch tail in `Dmax` rather than relying on that slack. -This is a timing contract rather than a cancellation or acquisition fence: the buffer prevents stale fallback results from passing that tracked Redis write only while the configured window remains active, and it does not force a fallback to read from an authoritative source. +This is a read-time timing contract rather than a cancellation or acquisition fence. The buffer makes covered frames unreadable; it does not stop their SETs, cancel in-flight work, or force fallback to read from an authoritative source. + +The version-1 value envelope, Redis keys, and decimal watermarks remain wire-format-compatible. The adapter contract and Lua surface are not source-compatible: `write()` is now void and has no `watermarkKey`; the stamp script, placeholder helpers, and `DialCacheRedisPlaceholderLostError` are removed; `dialcacheRedisScripts` and `DialCacheNodeRedisScripts` are removed because node-redis manages invalidation dispatch internally; `ValkeyGlideRuntime` no longer requires `ClusterBatch`; invalidation is the only script; `fill_blocked` is removed from `ShadowValidationOutcome`; and `tracked_ttl_clamped` is added to `MetricErrorKind`, so exhaustive switches and `Record` values must add it. The Prometheus future-timestamp histogram now uses a dedicated skew-oriented bucket schema, which is incompatible with an existing same-name collector registered with the former default buckets. Follow the externally gated [protocol cutover](#protocol-cutover) before deployment. Deploy the future-timestamp metric and external node-clock alerts first. The metric is only a workload-shaped smoke detector: it cannot detect co-skewed readers and writers, an ahead invalidator, watermark skew hidden by a fenced miss, or which node is wrong. Targeted invalidation is remote-only and enforced by Redis watermarks. `invalidateRemote` does not evict existing request-local or process-local entries. Strongly invalidated mutable data should disable request-local and process-local caching (or use a very short process-local TTL only when stale reads are acceptable). @@ -695,7 +707,7 @@ Coalescing only applies when at least one cache layer is active and the use case Because coalescing is keyed by the selected or direct key, concurrent calls with the same key share the leader's execution. Any function argument or captured value omitted from the key must be safe to share this way; include inputs such as locale, auth context, or cancellation behavior when they can change the returned value or whether the underlying loader should run separately. -The per-use-case `coalesce` boolean (default true) turns this sharing off. `coalesce: false` in a `defaultConfig` or runtime overlay disables both scopes: same-key concurrent callers each perform their own layer reads with their own full remote-read budget, their own fallback with an independent [fallback deadline](#fallback-deadlines), and their own cache writes — request-local and process-local publication is last-writer-wins, and each Redis write applies its ordinary TTL-based or watermark-fenced semantics. Request-local memoization of settled values still serves later sequential calls in the same scope. Use it when the key intentionally omits per-caller inputs that must not be shared, or when callers must not inherit a leader's failure or `FallbackTimeoutError`. Disabling coalescing reintroduces the thundering-herd exposure described above, emits `request`/`miss`/latency metrics once per caller instead of once per flight, never emits `dialcache_coalesced_counter`, and keeps `getCoalescingState()` idle for that use case. With shadow work enabled, each un-coalesced caller may attempt to schedule detached validation; same-key shadow deduplication and `shadowMaxInFlight` still bound admitted jobs and drop the excess, but source reads are no longer combined. +The per-use-case `coalesce` boolean (default true) turns this sharing off. `coalesce: false` in a `defaultConfig` or runtime overlay disables both scopes: same-key concurrent callers each perform their own layer reads with their own full remote-read budget, their own fallback with an independent [fallback deadline](#fallback-deadlines), and their own cache writes — request-local and process-local publication is last-writer-wins, every Redis write is one complete-frame last-writer-wins `SET`, and tracked Redis reads later apply their usual watermark fence. Request-local memoization of settled values still serves later sequential calls in the same scope. Use it when the key intentionally omits per-caller inputs that must not be shared, or when callers must not inherit a leader's failure or `FallbackTimeoutError`. Disabling coalescing reintroduces the thundering-herd exposure described above, emits `request`/`miss`/latency metrics once per caller instead of once per flight, never emits `dialcache_coalesced_counter`, and keeps `getCoalescingState()` idle for that use case. With shadow work enabled, each un-coalesced caller may attempt to schedule detached validation; same-key shadow deduplication and `shadowMaxInFlight` still bound admitted jobs and drop the excess, but source reads are no longer combined. ### Fallback deadlines @@ -791,11 +803,12 @@ The Prometheus adapter emits: | `dialcache_request_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `layer` | Cache-layer requests that reached an enabled layer | | `dialcache_miss_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `layer` | Cache misses | | `dialcache_disabled_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `layer`, `reason` | Cache skips (`context`, `policy_disabled`, `invalid_ttl`, `invalid_ramp`, `ramped_down`, `config_error`) | -| `dialcache_error_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `layer`, `error`, `in_fallback` | Cache/fallback errors classified by a bounded failure site | +| `dialcache_error_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `layer`, `error`, `in_fallback` | Cache/fallback errors and the bounded `tracked_ttl_clamped` configuration signal | | `dialcache_invalidation_counter` | Counter | `cache_namespace`, `key_type`, `layer` | Invalidation calls for the layers touched | | `dialcache_coalesced_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `scope` | Coalesced requests split by `request_local` or `process` scope | | `dialcache_shadow_validation_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `outcome` | Sampled Redis shadow-job outcomes | | `dialcache_shadow_value_age_histogram` | Histogram | `cache_namespace`, `use_case`, `key_type`, `outcome` | Age in seconds of the validated Redis value at shadow verdict time, recorded for `match` and `mismatch` | +| `dialcache_future_timestamp_offset_histogram` | Histogram | `cache_namespace`, `use_case`, `key_type`, `layer` | Positive offset in seconds for a valid tracked frame dated after the observing process clock | | `dialcache_compression_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `layer`, `outcome` | Payload compression outcomes: writes record `compressed`, `below_threshold`, `not_smaller`, or `write_over_limit`; reads record `decompressed`, `fallback_raw`, or `read_over_limit` | | `dialcache_get_timer` | Histogram | `cache_namespace`, `use_case`, `key_type`, `layer` | Cache get latency in seconds | | `dialcache_fallback_timer` | Histogram | `cache_namespace`, `use_case`, `key_type`, `layer` | Elapsed time until the underlying function settles or timeout rejection is delivered | @@ -805,6 +818,8 @@ The Prometheus adapter emits: | `dialcache_compression_ratio_histogram` | Histogram | `cache_namespace`, `use_case`, `key_type`, `layer` | Compressed-to-original payload size ratio for compressed writes | | `dialcache_compression_timer` | Histogram | `cache_namespace`, `use_case`, `key_type`, `layer`, `operation` | Payload compression and decompression latency in seconds | +The future-timestamp histogram uses dedicated buckets from millisecond-scale skew through multi-hour clock faults. It records one positive offset after a valid tracked frame is decoded. Caller-serving and initial shadow reads then reject that frame; confirmation reads retain it only for payload-equality classification. Invalid or non-finite timestamp values miss without entering histogram sums. The same future frame can be observed repeatedly. Alert against the deployment's allocated skew budget, not every millisecond-level sample. + `policy_disabled` means that a process-local or Redis layer has no effective TTL after runtime overlays are applied. It is an intentional policy outcome, including the default when `defaultConfig` is omitted, rather than a configuration-loading failure. Every metric carries `cache_namespace`, including disabled-context, key-construction, coalescing, shadow-validation, and invalidation paths that do not have a constructed key. Its value is `DialCacheConfig.namespace`, defaulting to `urn`. The `layer` label is `request_local`, `local` (process-local), `remote` (caller-serving Redis), or `remote_shadow` (detached, non-serving Redis work); `noop` means no cache layer was reached. Detached reads, serializer work, payload sizes, and Redis read/write errors use `remote_shadow`, while the dedicated bounded shadow `outcome` records the terminal job result. The bounded `scope` label on `dialcache_coalesced_counter` distinguishes request-local from instance-scoped single-flight work. `scope="process"` coordinates calls only within one `DialCache` instance; separate instances in the same process do not share in-flight state. @@ -843,7 +858,7 @@ dogStatsD.close(); `hot-shots` is the supported and tested client, but the adapter depends only on the exported `DatadogDogStatsDClient` structural interface. DialCache does not import or install `hot-shots`, create a client, flush buffers, close sockets, or otherwise own the client lifecycle. -`observationMetricType` is required. `"distribution"` is recommended when latency and size percentiles must aggregate across hosts; enable the desired distribution percentiles and aggregations in Datadog. Choose `"histogram"` when host-level histogram aggregation matches your existing Datadog setup. The choice applies uniformly to every duration, size, and ratio metric. Both modes produce Datadog custom metrics. Distribution volume scales with unique tag-value combinations: Datadog counts five baseline aggregations per combination, and enabling percentile aggregations adds five more. Review [Datadog's custom-metrics billing guidance](https://docs.datadoghq.com/account_management/billing/custom_metrics/) before rollout. Do not send both types under the same namespace: when changing types, use a new namespace during migration so one metric identity never mixes histogram and distribution points. +`observationMetricType` is required. `"distribution"` is recommended when latency and size percentiles must aggregate across hosts; enable the desired distribution percentiles and aggregations in Datadog. Choose `"histogram"` when host-level histogram aggregation matches your existing Datadog setup. The choice applies uniformly to every observation metric, including durations, sizes, ratios, ages, and timestamp offsets. Both modes produce Datadog custom metrics. Distribution volume scales with unique tag-value combinations: Datadog counts five baseline aggregations per combination, and enabling percentile aggregations adds five more. Review [Datadog's custom-metrics billing guidance](https://docs.datadoghq.com/account_management/billing/custom_metrics/) before rollout. Do not send both types under the same namespace: when changing types, use a new namespace during migration so one metric identity never mixes histogram and distribution points. `DatadogMetricsOptions.namespace` is the metric-name namespace and defaults to `dialcache`. It is separate from `DialCacheConfig.namespace`, the logical cache namespace emitted as the `cache_namespace` tag. The Datadog metric namespace must start with a letter and contain only letters, numbers, underscores, and dot-separated non-empty segments. The adapter rejects invalid metric namespaces and final metric names longer than 200 characters rather than relying on client-side normalization. A `hot-shots` `prefix` is applied after the adapter constructs the name, so include that prefix when checking the final length and avoid combining it with the metric namespace accidentally. Client-level `globalTags` are appended by `hot-shots`; the table below lists the tags added by the adapter. @@ -854,11 +869,12 @@ The Datadog adapter emits exact increments of `1` for counters and preserves sec | `dialcache.request.count` | Count | `cache_namespace`, `use_case`, `key_type`, `layer` | Cache-layer requests that reached an enabled layer | | `dialcache.miss.count` | Count | `cache_namespace`, `use_case`, `key_type`, `layer` | Cache misses | | `dialcache.disabled.count` | Count | `cache_namespace`, `use_case`, `key_type`, `layer`, `reason` | Cache skips by bounded reason | -| `dialcache.error.count` | Count | `cache_namespace`, `use_case`, `key_type`, `layer`, `error`, `in_fallback` | Cache/fallback errors by bounded failure site | +| `dialcache.error.count` | Count | `cache_namespace`, `use_case`, `key_type`, `layer`, `error`, `in_fallback` | Cache/fallback errors and the bounded `tracked_ttl_clamped` configuration signal | | `dialcache.invalidation.count` | Count | `cache_namespace`, `key_type`, `layer` | Invalidation calls for the layers touched | | `dialcache.coalesced.count` | Count | `cache_namespace`, `use_case`, `key_type`, `scope` | Coalesced requests by sharing scope | | `dialcache.shadow.count` | Count | `cache_namespace`, `use_case`, `key_type`, `outcome` | Sampled Redis shadow-job outcomes | | `dialcache.shadow.value_age` | Distribution or histogram | `cache_namespace`, `use_case`, `key_type`, `outcome` | Age in seconds of the validated Redis value at shadow verdict time, recorded for `match` and `mismatch` | +| `dialcache.future_timestamp_offset` | Distribution or histogram | `cache_namespace`, `use_case`, `key_type`, `layer` | Positive offset in seconds for a valid tracked frame dated after the observing process clock | | `dialcache.compression.count` | Count | `cache_namespace`, `use_case`, `key_type`, `layer`, `outcome` | Payload compression outcomes: writes record `compressed`, `below_threshold`, `not_smaller`, or `write_over_limit`; reads record `decompressed`, `fallback_raw`, or `read_over_limit` | | `dialcache.get.duration` | Distribution or histogram | `cache_namespace`, `use_case`, `key_type`, `layer` | Cache get latency in seconds | | `dialcache.fallback.duration` | Distribution or histogram | `cache_namespace`, `use_case`, `key_type`, `layer` | Elapsed time until the underlying function settles or timeout rejection is delivered | @@ -880,7 +896,7 @@ The `error` label reports where an operation failed rather than copying the thro | `config_resolution` | Runtime or layer configuration, or ramp resolution, failed | | `cache_read` | A local-cache or Redis read failed | | `cache_read_timeout` | A Redis read exceeded its effective remote-read deadline | -| `cache_write` | A local-cache or Redis write failed; tracked Redis writes add a benign self-healing floor under same-key contention (see [Redis-backed TTL cache](#redis-backed-ttl-cache)) | +| `cache_write` | A local-cache or Redis write failed | | `serialization_load` | Deserializing a Redis payload failed | | `serialization_dump` | Serializing a value for Redis failed | | `compression` | zstd compression failed while preparing a Redis write | @@ -892,7 +908,7 @@ These values are defined by the backend-neutral core and are identical for every ### Custom adapters -For other telemetry backends, implement `DialCacheMetricsAdapter` and pass the adapter through `new DialCache({ metrics })`. Every backend-neutral label object exposes the logical namespace as camel-case `cacheNamespace`; adapters should map it to their backend's `cache_namespace` label/tag. This field is present even when no key or cache layer was reached. Implement the optional `shadowValidation` method to enable shadow work as well as record its outcomes; omitting it leaves all shadow work disabled even when `shadow.ramp` is nonzero or mismatch logging is enabled. The optional `observeShadowValueAge` method records the validated value's age in seconds for `match` and `mismatch` outcomes; omitting it skips only that observation without affecting shadow eligibility. Every metrics callback is fire-and-forget: DialCache isolates synchronous throws and consumes rejections from returned promises or thenables, but never awaits or drains observer work. Omit `metrics` to disable metrics. +For other telemetry backends, implement `DialCacheMetricsAdapter` and pass the adapter through `new DialCache({ metrics })`. Every backend-neutral label object exposes the logical namespace as camel-case `cacheNamespace`; adapters should map it to their backend's `cache_namespace` label/tag. This field is present even when no key or cache layer was reached. Implement the optional `shadowValidation` method to enable shadow work as well as record its outcomes; omitting it leaves all shadow work disabled even when `shadow.ramp` is nonzero or mismatch logging is enabled. The optional `observeShadowValueAge` method records the validated value's age in seconds for `match` and `mismatch` outcomes; omitting it skips only that observation without affecting shadow eligibility. The optional `observeFutureTimestampOffset` method receives existing bounded cache labels plus the exact positive offset in seconds; omitting it does not change tracked read decisions: serving and initial-shadow reads still miss, while confirmation reads still retain the frame for payload comparison. Every metrics callback is fire-and-forget: DialCache isolates synchronous throws and consumes rejections from returned promises or thenables, but never awaits or drains observer work. Omit `metrics` to disable metrics. ## Maintainers @@ -904,17 +920,17 @@ From a repository checkout, run the semantic microbenchmark after installing dep pnpm benchmark:request-local ``` -The command builds `dist` before reporting ten scenarios: sequential request-local hits, sequential process-local hits, enabled bounded fallbacks, request-local coalescing fan-out, process coalescing fan-out, remote-read-deadline coalescing fan-out, tracked Redis hits with shadow omitted, tracked Redis hits deterministically outside a partial shadow ramp, a ramped-down warm-hit confirmation, and a ramped-down clean-miss fill. Both shadow scenarios prove that the caller completes before detached Redis work. The benchmark is a maintainer tool and is not included in the published package. It asserts fallback counts, Redis behavior, coalescing state, timer cleanup, returned values, exactly-once SoT reuse, and conditional confirmation/fill without applying a timing threshold. Override its work sizes with `DIALCACHE_BENCH_ITERATIONS` and `DIALCACHE_BENCH_FANOUT`. +The command builds `dist` before reporting ten scenarios: sequential request-local hits, sequential process-local hits, enabled bounded fallbacks, request-local coalescing fan-out, process coalescing fan-out, remote-read-deadline coalescing fan-out, tracked Redis hits with shadow omitted, tracked Redis hits deterministically outside a partial shadow ramp, a ramped-down warm-hit confirmation, and a ramped-down semantic-miss fill. Both shadow scenarios prove that the caller completes before detached Redis work. The benchmark is a maintainer tool and is not included in the published package. It asserts fallback counts, Redis behavior, coalescing state, timer cleanup, returned values, exactly-once SoT reuse, and conditional confirmation/fill without applying a timing threshold. Override its work sizes with `DIALCACHE_BENCH_ITERATIONS` and `DIALCACHE_BENCH_FANOUT`. ### Redis write benchmark -With a Redis reachable at `REDIS_URL` (default `redis://127.0.0.1:6379`, e.g. `docker run --rm -p 6379:6379 redis:6.2`), measure the local build's write path: +With an otherwise idle Redis reachable at `REDIS_URL` (default `redis://127.0.0.1:6379`, e.g. `docker run --rm -p 6379:6379 redis:6.2`), measure the local build's write path. The benchmark resets global command statistics between cases, so use a disposable or dedicated instance: ```bash pnpm benchmark:redis-write ``` -The command builds `dist`, then runs sequential tracked and untracked writes at 100 B, 10 KiB, 100 KiB, and 1 MiB payloads, reporting server-side command cost per write from `INFO commandstats` (the `EVALSHA` entry envelopes the stamp script's internal calls) and client-side p50/p95 latency. Like the cache-path benchmark it is a maintainer tool, is not part of the published package, and asserts no timing thresholds — absolute numbers depend on the machine, engine, and load, so compare runs only within one environment. Scale iteration counts with `DIALCACHE_BENCH_WRITE_SCALE`. +The command builds `dist`, then runs sequential native writes at 100 B, 10 KiB, 100 KiB, and 1 MiB payloads. It reports `SET`, script, and `TIME` calls per operation, server-side `SET` cost from `INFO commandstats`, and client-side p50/p95 latency. Semantic assertions require exactly one `SET`, zero scripts, and zero `TIME` calls per write. Because operations are sequential, the benchmark validates command shape and single-operation latency; it does not measure saturated concurrent throughput or maximum write capacity. Like the cache-path benchmark it is a maintainer tool, is not part of the published package, and applies no timing threshold — absolute numbers depend on the machine, engine, and load, so compare runs only within one environment. Scale iteration counts with `DIALCACHE_BENCH_WRITE_SCALE`. ### Releasing diff --git a/scripts/benchmark-redis-write.mjs b/scripts/benchmark-redis-write.mjs index 7a05fe7..53cc6ed 100644 --- a/scripts/benchmark-redis-write.mjs +++ b/scripts/benchmark-redis-write.mjs @@ -1,17 +1,18 @@ // Maintainer benchmark for the Redis write path. Measures the local build's -// tracked and untracked writes against a live Redis and reports server-side -// command cost per write (INFO commandstats; the EVALSHA entry envelopes -// script-internal calls) alongside client-side latency percentiles. It -// asserts nothing and applies no timing thresholds: absolute numbers are -// machine-, engine-, and load-dependent, so compare runs only against the -// same environment. +// writes against a live Redis and reports server-side command cost per write +// (INFO commandstats) alongside client-side latency percentiles. It asserts +// the steady-state one-SET shape and that no write invokes Lua or Redis TIME, +// but applies no timing thresholds: absolute numbers are machine-, engine-, +// and load-dependent, so compare runs only against the same idle Redis. // // Requires a reachable Redis, e.g.: docker run --rm -p 6379:6379 redis:6.2 // Usage: pnpm benchmark:redis-write (REDIS_URL to override) // DIALCACHE_BENCH_WRITE_SCALE scales iteration counts (default 1). +import assert from "node:assert/strict"; + import { createClient } from "redis"; -import { createNodeRedisDialCacheClient, dialcacheRedisScripts } from "../dist/node-redis.js"; +import { createNodeRedisDialCacheClient } from "../dist/node-redis.js"; const REDIS_URL = process.env.REDIS_URL ?? "redis://127.0.0.1:6379"; const SCALE = Number(process.env.DIALCACHE_BENCH_WRITE_SCALE ?? "1"); @@ -42,7 +43,6 @@ async function commandStats(client) { const client = createClient({ url: REDIS_URL, - scripts: dialcacheRedisScripts, disableOfflineQueue: true, socket: { connectTimeout: 2_000 }, }); @@ -56,57 +56,58 @@ try { const adapter = createNodeRedisDialCacheClient(client); const rows = []; -for (const mode of ["tracked", "untracked"]) { - for (const size of SIZES) { - const iterations = Math.max(1, Math.round(size.n * SCALE)); - const payload = "x".repeat(size.bytes); - const valueKey = `benchmark:write:${mode}:${size.bytes}:value`; - const watermarkKey = `benchmark:write:${mode}:${size.bytes}:watermark`; - const request = mode === "tracked" - ? { valueKey, watermarkKey, cacheTtlMs: 60_000, value: payload } - : { valueKey, cacheTtlMs: 60_000, value: payload }; - - for (let i = 0; i < WARMUP; i += 1) { - await adapter.write(request); - } - await client.sendCommand(["CONFIG", "RESETSTAT"]); +for (const size of SIZES) { + const iterations = Math.max(1, Math.round(size.n * SCALE)); + const payload = "x".repeat(size.bytes); + const valueKey = `benchmark:write:native:${size.bytes}:value`; + const request = { valueKey, cacheTtlMs: 60_000, value: payload }; - const latenciesUsec = []; - for (let i = 0; i < iterations; i += 1) { - const start = process.hrtime.bigint(); - await adapter.write(request); - latenciesUsec.push(Number(process.hrtime.bigint() - start) / 1_000); - } + for (let i = 0; i < WARMUP; i += 1) { + await adapter.write(request); + } + await client.sendCommand(["CONFIG", "RESETSTAT"]); - // Sum only the commands the client dispatches top-level (SET, EVALSHA, - // and the EVAL recovery). Script-internal calls surface in commandstats - // too, but the EVALSHA entry already envelopes their execution time. - const stats = await commandStats(client); - const serverUsec = (stats.set?.usec ?? 0) - + (stats.evalsha?.usec ?? 0) - + (stats.eval?.usec ?? 0); - latenciesUsec.sort((a, b) => a - b); - rows.push({ - mode, - size: size.name, - writes: iterations, - serverUsecPerWrite: serverUsec / iterations, - clientP50Usec: percentile(latenciesUsec, 50), - clientP95Usec: percentile(latenciesUsec, 95), - }); + const latenciesUsec = []; + for (let i = 0; i < iterations; i += 1) { + const start = process.hrtime.bigint(); + await adapter.write(request); + latenciesUsec.push(Number(process.hrtime.bigint() - start) / 1_000); } + + const stats = await commandStats(client); + const setCalls = stats.set?.calls ?? 0; + const scriptCalls = (stats.evalsha?.calls ?? 0) + (stats.eval?.calls ?? 0); + const timeCalls = stats.time?.calls ?? 0; + assert.equal(setCalls, iterations, "writes must issue one top-level SET each"); + assert.equal(scriptCalls, 0, "writes must not dispatch Lua scripts"); + assert.equal(timeCalls, 0, "writes must not invoke Redis TIME"); + const serverUsec = stats.set?.usec ?? 0; + latenciesUsec.sort((a, b) => a - b); + rows.push({ + size: size.name, + writes: iterations, + setCallsPerWrite: setCalls / iterations, + scriptCallsPerWrite: scriptCalls / iterations, + timeCallsPerWrite: timeCalls / iterations, + serverUsecPerWrite: serverUsec / iterations, + clientP50Usec: percentile(latenciesUsec, 50), + clientP95Usec: percentile(latenciesUsec, 95), + }); } await client.quit(); console.log(`Redis write benchmark — ${REDIS_URL}`); -console.log("mode size writes server µs/write client p50 µs client p95 µs"); +console.log("size writes SET/op script/op TIME/op server µs/write client p50 µs client p95 µs"); for (const row of rows) { console.log( - row.mode.padEnd(10) - + row.size.padEnd(10) + row.size.padEnd(10) + String(row.writes).padEnd(9) + + row.setCallsPerWrite.toFixed(1).padEnd(9) + + row.scriptCallsPerWrite.toFixed(1).padEnd(12) + + row.timeCallsPerWrite.toFixed(1).padEnd(10) + row.serverUsecPerWrite.toFixed(1).padEnd(18) + row.clientP50Usec.toFixed(0).padEnd(16) + row.clientP95Usec.toFixed(0), ); } +console.log("Command-shape assertions passed; elapsed times are informational and have no pass/fail threshold."); diff --git a/scripts/benchmark-request-local.mjs b/scripts/benchmark-request-local.mjs index 8a0c3b9..96bbeb0 100644 --- a/scripts/benchmark-request-local.mjs +++ b/scripts/benchmark-request-local.mjs @@ -248,11 +248,9 @@ async function benchmarkRedisReadDeadlineCoalescing(fanout) { redisReadCalls += 1; started.resolve(); await gate.promise; - return JSON.stringify("shared"); - }, - async write() { - return true; + return { payload: JSON.stringify("shared"), createdAtMs: Date.now() }; }, + async write() {}, async invalidate() {}, }; const dialcache = new DialCache({ @@ -318,15 +316,15 @@ async function benchmarkSequentialTrackedRedisHits(iterations, { scenario, useCa let redisReadCalls = 0; let redisWriteCalls = 0; let redisInvalidationCalls = 0; + const frame = { payload: JSON.stringify("shared"), createdAtMs: Date.now() }; const redisClient = { async read({ watermarkKey }) { assert.equal(typeof watermarkKey, "string", "the benchmark must exercise tracked Redis reads"); redisReadCalls += 1; - return JSON.stringify("shared"); + return frame; }, async write() { redisWriteCalls += 1; - return true; }, async invalidate() { redisInvalidationCalls += 1; @@ -399,7 +397,6 @@ async function benchmarkDarkShadowDetachment() { }, async write() { redisWriteCalls += 1; - return true; }, async invalidate() { redisInvalidationCalls += 1; @@ -451,7 +448,7 @@ async function benchmarkDarkShadowDetachment() { assert.equal(redisReadCalls, 1, "the detached C0 read should have started"); assert.equal(fallbackCalls, 1, "the caller and shadow validation must share one SoT invocation"); - readGate.resolve(JSON.stringify(cachedValue)); + readGate.resolve({ payload: JSON.stringify(cachedValue), createdAtMs: Date.now() }); await nextTurn(); assert.equal(await outcomeGate.promise, "mismatch"); assert.equal(redisReadCalls, 2, "only a mismatch candidate should add confirmation C1"); @@ -481,12 +478,15 @@ async function benchmarkDarkShadowFillDetachment() { redisReadCalls += 1; return null; }, - async write({ watermarkKey }) { - assert.equal(typeof watermarkKey, "string", "dark shadow fills must remain tracked"); + async write(request) { + assert.equal( + Object.hasOwn(request, "watermarkKey"), + false, + "dark shadow fills must use the unified native write request", + ); redisWriteCalls += 1; writeStarted.resolve(); await writeGate.promise; - return true; }, async invalidate() {}, }; diff --git a/scripts/test-package.mjs b/scripts/test-package.mjs index 03b00ed..2979455 100644 --- a/scripts/test-package.mjs +++ b/scripts/test-package.mjs @@ -9,8 +9,76 @@ const exec = promisify(execFile); const root = dirname(dirname(fileURLToPath(import.meta.url))); const workspace = await mkdtemp(join(tmpdir(), "dialcache-package-")); const fallbackTimeoutMarker = "dialcache-fallback-timeout-delivered"; +const nodeInvalidationMarker = "dialcache-node-invalidation-retry-verified"; const observerIsolationMarker = "dialcache-observer-rejections-isolated"; const shadowPayloadReleaseMarker = "dialcache-shadow-payload-released"; +const packedInvalidationCheckSource = String.raw` +function createPackedNodeRedisInvalidationAdapter(nodeRedis, dispatch, label) { + const client = { + get: async () => null, + sendCommand: async (...callArgs) => { + if ( + callArgs.length !== 2 + || !Array.isArray(callArgs[0]) + || callArgs[1]?.returnBuffers !== true + ) { + throw new Error("The packed " + label + " adapter used the wrong standalone command shape"); + } + return await dispatch(callArgs[0]); + }, + }; + return nodeRedis.createNodeRedisDialCacheClient(client); +} + +function createPackedGlideInvalidationAdapter(glide, appGlide, dispatch) { + const client = { + customCommand: async (args) => await dispatch(args), + }; + const runtime = { + ...appGlide, + GlideClient: { [Symbol.hasInstance]: (value) => value === client }, + GlideClusterClient: { [Symbol.hasInstance]: () => false }, + }; + return glide.createValkeyGlideDialCacheClient(client, runtime); +} + +async function verifyPackedInvalidation({ createAdapter, label, redisProtocol }) { + const dispatches = []; + const dispatch = async (args) => { + dispatches.push(args); + if (dispatches.length === 1) { + throw new Error("packed invalidation EVALSHA rejected"); + } + return 1; + }; + const invalidatedAtMs = 1700000000456; + const nativeDateNow = Date.now; + Date.now = () => invalidatedAtMs; + try { + await createAdapter(dispatch).invalidate({ + watermarkKey: "tracked:{id}:watermark", + futureBufferMs: 50, + }); + } finally { + Date.now = nativeDateNow; + } + + const script = redisProtocol.INVALIDATE_CACHE_SCRIPT; + const sha = createHash("sha1").update(script).digest("hex"); + const args = ["1", "tracked:{id}:watermark", "50", String(invalidatedAtMs)]; + const expected = [ + ["EVALSHA", sha, ...args], + ["EVAL", script, ...args], + ]; + const commandsMatch = dispatches.length === expected.length + && dispatches.every((command, index) => + command.length === expected[index].length + && command.every((part, partIndex) => part === expected[index][partIndex])); + if (!commandsMatch) { + throw new Error("The packed " + label + " invalidation script or argument contract is invalid"); + } +} +`; const rootConsumer = `import { CacheLayer, DialCache, @@ -52,21 +120,28 @@ const rootConsumer = `import { } from "dialcache"; // @ts-expect-error The unused MissingKeyConfigError class was removed instead of deprecated. import { MissingKeyConfigError } from "dialcache"; +// @ts-expect-error Placeholder promotion was removed from the Redis adapter protocol. import { DialCacheRedisPlaceholderLostError } from "dialcache"; -import { createNodeRedisDialCacheClient, dialcacheRedisScripts } from "dialcache/node-redis"; +import { createNodeRedisDialCacheClient } from "dialcache/node-redis"; +// @ts-expect-error The node-redis adapter no longer requires public script registrations. +import { dialcacheRedisScripts } from "dialcache/node-redis"; +// @ts-expect-error The node-redis script-registration type was removed with the facade. +import type { DialCacheNodeRedisScripts } from "dialcache/node-redis"; import { ceilSupportedCacheTtlMs, decodeRedisFrame, decodeTrackedRedisFrame, encodeRedisFrame, - encodeTrackedRedisPlaceholder, - resolveTrackedRedisWriteReply, validateRedisScriptInvalidationReply, validateRedisSetReply, - WRITE_TRACKED_STAMP_SCRIPT, type DecodedRedisFrame, - type TrackedRedisPlaceholder, } from "dialcache/redis-protocol"; +// @ts-expect-error Placeholder promotion was removed from the Redis adapter protocol. +import { encodeTrackedRedisPlaceholder } from "dialcache/redis-protocol"; +// @ts-expect-error Tracked writes no longer have a script reply to resolve. +import { resolveTrackedRedisWriteReply } from "dialcache/redis-protocol"; +// @ts-expect-error Tracked writes are native SET commands and no longer use Lua. +import { WRITE_TRACKED_STAMP_SCRIPT } from "dialcache/redis-protocol"; // @ts-expect-error The codec functions replaced the frame-version wire constant. import { REDIS_FRAME_VERSION } from "dialcache/redis-protocol"; // @ts-expect-error The codec functions replaced the UTF-8 encoding wire constant. @@ -79,7 +154,7 @@ import { READ_CACHE_SCRIPT } from "dialcache/redis-protocol"; import { READ_TRACKED_CACHE_SCRIPT } from "dialcache/redis-protocol"; // @ts-expect-error The untracked write Lua was replaced by a native client-framed SET. import { WRITE_CACHE_SCRIPT } from "dialcache/redis-protocol"; -// @ts-expect-error The tracked write Lua was replaced by a native SET plus the stamp script. +// @ts-expect-error The tracked write Lua was replaced by a native client-framed SET. import { WRITE_TRACKED_CACHE_SCRIPT } from "dialcache/redis-protocol"; import { DatadogDialCacheMetrics, @@ -112,6 +187,12 @@ const metrics: DialCacheMetricsAdapter = { }; const shadowMetrics: DialCacheMetricsAdapter = { ...metrics, + observeFutureTimestampOffset: (labels: CacheMetricLabels, seconds: number) => { + const cacheNamespace: string = labels.cacheNamespace; + const offsetSeconds: number = seconds; + void cacheNamespace; + void offsetSeconds; + }, shadowValidation: (labels: ShadowValidationMetricLabels) => { const outcome: ShadowValidationOutcome = labels.outcome; void outcome; @@ -122,7 +203,6 @@ const shadowOutcomes: Readonly> = { mismatch: true, superseded: true, filled: true, - fill_blocked: true, fill_error: true, redis_error: true, source_error: true, @@ -176,20 +256,10 @@ const decodedStaleRedisFrame: DecodedRedisFrame | null = decodeTrackedRedisFrame emptyRedisFrame, Buffer.from("1"), ); -const placeholderRedisFrame: Buffer = encodeRedisFrame("pending", 0); -const trackedRedisPlaceholder: TrackedRedisPlaceholder = encodeTrackedRedisPlaceholder("pending"); -const stampReplyResolution: boolean = resolveTrackedRedisWriteReply(1); +const zeroTimestampRedisFrame: Buffer = encodeRedisFrame("pending", 0); const setReplyValidation: void = validateRedisSetReply("OK"); const invalidationReplyValidation: 1 = validateRedisScriptInvalidationReply(1); const ceiledCacheTtlMs: number = ceilSupportedCacheTtlMs(1_000.5); -const placeholderLostError = new DialCacheRedisPlaceholderLostError("lost"); -const stampScriptSource: string = WRITE_TRACKED_STAMP_SCRIPT; -const stampArguments: Array = dialcacheRedisScripts.dialcacheWriteTrackedStamp.transformArguments( - "tracked:{id}:value", - "tracked:{id}:watermark", - 1_000, - trackedRedisPlaceholder.nonce, -); const fallbackTimeoutError = new FallbackTimeoutError("Load", 1_000); const redisReadTimeoutError = new RedisReadTimeoutError("Load", 100); const coalescingState: CoalescingState = cache.getCoalescingState(); @@ -364,6 +434,7 @@ const metricErrorKinds: Readonly> = { cache_read: true, cache_read_timeout: true, cache_write: true, + tracked_ttl_clamped: true, serialization_load: true, serialization_dump: true, compression: true, @@ -403,7 +474,9 @@ const unboundedCompressionOutcome: CompressionOutcome = "inflated"; const customRedisClient: DialCacheRedisClient = { // The optional second read argument preserves one-argument custom clients. read: async () => ({ payload: Buffer.from([0, 255]), createdAtMs: 1 }), - write: async ({ value }) => typeof value === "string" || Buffer.isBuffer(value), + write: async ({ value }) => { + void (typeof value === "string" || Buffer.isBuffer(value)); + }, invalidate: async () => undefined, }; const redisClientMethods: Readonly> = { @@ -419,20 +492,25 @@ const redisConfigAcceptsCompressionOptOut: RedisConfig = { const cacheHasNoFlushAll: "flushAll" extends keyof DialCache ? false : true = true; const cacheHasNoClose: "close" extends keyof DialCache ? false : true = true; const clientHasNoFlushAll: "flushAll" extends keyof DialCacheRedisClient ? false : true = true; -type TrackedRedisWriteRequest = Extract; -const trackedWriteHasNoWatermarkTtlFloor: "watermarkTtlFloorMs" extends keyof TrackedRedisWriteRequest +const writeHasNoWatermark: "watermarkKey" extends keyof RedisWriteRequest ? false : true = true; +const trackedWriteHasNoWatermarkTtlFloor: "watermarkTtlFloorMs" extends keyof RedisWriteRequest + ? false + : true = true; +const trackedWriteHasNoCreatedAt: "createdAtMs" extends keyof RedisWriteRequest ? false : true = true; const invalidationHasNoWatermarkTtlFloor: "watermarkTtlFloorMs" extends keyof RedisInvalidationRequest ? false : true = true; +const invalidationHasNoInvalidatedAt: "invalidatedAtMs" extends keyof RedisInvalidationRequest + ? false + : true = true; const legacyTrackedWriteRequest: RedisWriteRequest = { valueKey: "tracked:{id}:value", + // @ts-expect-error Writes no longer inspect or mutate invalidation watermarks. watermarkKey: "tracked:{id}:watermark", cacheTtlMs: 1_000, value: "tracked", - // @ts-expect-error Watermark lifetime is derived by the Redis invalidation protocol. - watermarkTtlFloorMs: 1_000, }; const legacyInvalidationRequest: RedisInvalidationRequest = { watermarkKey: "tracked:{id}:watermark", @@ -530,29 +608,21 @@ void redisConfigAcceptsCompressionOptOut; void createNodeRedisDialCacheClient; void decodedEmptyRedisFrame; void decodedStaleRedisFrame; -// @ts-expect-error Native reads removed the legacy node-redis registration. -void dialcacheRedisScripts.dialcacheRead; -// @ts-expect-error Native tracked reads removed the legacy node-redis registration. -void dialcacheRedisScripts.dialcacheReadTracked; -// @ts-expect-error Native SET writes removed the legacy node-redis registration. -void dialcacheRedisScripts.dialcacheWrite; -// @ts-expect-error The stamp protocol removed the legacy tracked-write registration. -void dialcacheRedisScripts.dialcacheWriteTracked; -void dialcacheRedisScripts.dialcacheWriteTrackedStamp; +void dialcacheRedisScripts; +void (undefined as unknown as DialCacheNodeRedisScripts); void READ_CACHE_SCRIPT; void READ_TRACKED_CACHE_SCRIPT; void WRITE_CACHE_SCRIPT; void WRITE_TRACKED_CACHE_SCRIPT; -void placeholderRedisFrame; -void trackedRedisPlaceholder; -void stampReplyResolution; +void zeroTimestampRedisFrame; void setReplyValidation; -void placeholderLostError; +void DialCacheRedisPlaceholderLostError; +void encodeTrackedRedisPlaceholder; +void resolveTrackedRedisWriteReply; void REDIS_FRAME_VERSION; void REDIS_ENCODING_UTF8; void REDIS_ENCODING_BINARY; -void stampScriptSource; -void stampArguments; +void WRITE_TRACKED_STAMP_SCRIPT; void customRedisClient; const globalSerializer: Serializer = { dump: () => "global", @@ -568,8 +638,11 @@ cacheWithGlobalSerializer.getOrLoad(async () => new Date(0), inlineOptionsFor("G void cacheHasNoFlushAll; void cacheHasNoClose; void clientHasNoFlushAll; +void writeHasNoWatermark; void trackedWriteHasNoWatermarkTtlFloor; +void trackedWriteHasNoCreatedAt; void invalidationHasNoWatermarkTtlFloor; +void invalidationHasNoInvalidatedAt; void legacyTrackedWriteRequest; void legacyInvalidationRequest; void configHasNoMetricsRegistry; @@ -622,7 +695,7 @@ import { import { type ValkeyGlideDialCacheClient } from "dialcache/valkey-glide"; // @ts-expect-error The handle-free GLIDE adapter removed the Script handle type. import { type ValkeyGlideScriptHandle } from "dialcache/valkey-glide"; -import { createNodeRedisDialCacheClient, dialcacheRedisScripts } from "dialcache/node-redis"; +import { createNodeRedisDialCacheClient } from "dialcache/node-redis"; import { Registry, type OpenMetricsContentType } from "prom-client"; const registry = new Registry(); @@ -635,14 +708,19 @@ openMetricsRegistry.setContentType(Registry.OPENMETRICS_CONTENT_TYPE); const openMetricsAdapter = new PrometheusDialCacheMetrics({ registry: openMetricsRegistry, prefix: "open_" }); const registryIsRequired: {} extends Pick ? false : true = true; const glideRedisClient: DialCacheRedisClient | undefined = undefined; -const standaloneNodeRedisClient = createRedisClient({ scripts: dialcacheRedisScripts }); +const standaloneNodeRedisClient = createRedisClient(); const clusterNodeRedisClient = createRedisCluster({ rootNodes: [{ url: "redis://127.0.0.1:6379" }], - scripts: dialcacheRedisScripts, }); const standaloneNodeRedisAdapter = createNodeRedisDialCacheClient(standaloneNodeRedisClient); const clusterNodeRedisAdapter = createNodeRedisDialCacheClient(clusterNodeRedisClient); const glideRuntime: ValkeyGlideRuntime = valkeyGlide; +const glideRuntimeWithoutClusterBatch: ValkeyGlideRuntime = { + Batch: valkeyGlide.Batch, + GlideClient: valkeyGlide.GlideClient, + GlideClusterClient: valkeyGlide.GlideClusterClient, + Decoder: valkeyGlide.Decoder, +}; declare const standaloneGlideClient: valkeyGlide.GlideClient; declare const clusterGlideClient: valkeyGlide.GlideClusterClient; const standaloneGlideAdapter: DialCacheRedisClient = createValkeyGlideDialCacheClient(standaloneGlideClient, glideRuntime); @@ -671,6 +749,7 @@ void classAdapter; void openMetricsAdapter; void registryIsRequired; void glideRedisClient; +void glideRuntimeWithoutClusterBatch; void standaloneNodeRedisAdapter; void clusterNodeRedisAdapter; void standaloneGlideAdapter; @@ -722,19 +801,26 @@ try { [ "--input-type=module", "--eval", - `const root = await import("dialcache"); + `const { createHash } = await import("node:crypto"); +const root = await import("dialcache"); const nodeRedis = await import("dialcache/node-redis"); await import("dialcache/valkey-glide"); await import("dialcache/datadog"); const redisProtocol = await import("dialcache/redis-protocol"); -// Each bundle embeds its own copy of the Lua sources; a divergence forks the -// protocol (different SHA1s) without failing any behavioral test. -if ( - nodeRedis.dialcacheRedisScripts.dialcacheWriteTrackedStamp.SCRIPT !== redisProtocol.WRITE_TRACKED_STAMP_SCRIPT - || nodeRedis.dialcacheRedisScripts.dialcacheInvalidate.SCRIPT !== redisProtocol.INVALIDATE_CACHE_SCRIPT -) { - throw new Error("The packed ESM node-redis Lua sources diverged from the redis-protocol entry"); -} +${packedInvalidationCheckSource} +if (typeof nodeRedis.createNodeRedisDialCacheClient !== "function") { + throw new Error("The packed ESM node-redis adapter export is missing"); +} +if ("dialcacheRedisScripts" in nodeRedis) { + throw new Error("The removed ESM node-redis script-registration facade is still exported"); +} +await verifyPackedInvalidation({ + createAdapter: (dispatch) => + createPackedNodeRedisInvalidationAdapter(nodeRedis, dispatch, "ESM node-redis"), + label: "ESM node-redis", + redisProtocol, +}); +console.log("${nodeInvalidationMarker}"); const fallbackTimeoutError = new root.FallbackTimeoutError("PackageRuntime", 1000); if (!(fallbackTimeoutError instanceof root.DialCacheError) || fallbackTimeoutError.timeoutMs !== 1000) { throw new Error("The root ESM fallback-timeout error export is invalid"); @@ -765,22 +851,8 @@ try { } console.log("${fallbackTimeoutMarker}"); } -try { - nodeRedis.dialcacheRedisScripts.dialcacheWriteTrackedStamp.transformReply(3); - throw new Error("Expected an invalid node-redis script reply to fail"); -} catch (error) { - if (!(error instanceof root.DialCacheRedisProtocolError)) { - throw new Error("The node-redis protocol error does not match the root ESM export"); - } -} -if ("MissingKeyConfigError" in root) { - throw new Error("The removed MissingKeyConfigError class must not be exported from the root ESM entry"); -} -if ( - "dialcacheRead" in nodeRedis.dialcacheRedisScripts - || "dialcacheReadTracked" in nodeRedis.dialcacheRedisScripts -) { - throw new Error("The removed read scripts must not be registered by the packed ESM node-redis entry"); +if ("MissingKeyConfigError" in root || "DialCacheRedisPlaceholderLostError" in root) { + throw new Error("Removed error classes must not be exported from the root ESM entry"); } if ( "READ_CACHE_SCRIPT" in redisProtocol @@ -788,36 +860,28 @@ if ( ) { throw new Error("The removed read scripts must not be exported by the packed ESM Redis protocol entry"); } -if ( - "dialcacheWrite" in nodeRedis.dialcacheRedisScripts - || "dialcacheWriteTracked" in nodeRedis.dialcacheRedisScripts -) { - throw new Error("The removed write scripts must not be registered by the packed ESM node-redis entry"); -} if ( "WRITE_CACHE_SCRIPT" in redisProtocol || "WRITE_TRACKED_CACHE_SCRIPT" in redisProtocol ) { throw new Error("The removed write scripts must not be exported by the packed ESM Redis protocol entry"); } -if (typeof redisProtocol.WRITE_TRACKED_STAMP_SCRIPT !== "string") { - throw new Error("The packed ESM Redis protocol entry must export the tracked stamp script source"); +if ( + "WRITE_TRACKED_STAMP_SCRIPT" in redisProtocol + || "encodeTrackedRedisPlaceholder" in redisProtocol + || "resolveTrackedRedisWriteReply" in redisProtocol +) { + throw new Error("Removed placeholder and stamp helpers must not be exported by the packed ESM Redis protocol entry"); } const esmRoundTrip = redisProtocol.decodeRedisFrame(redisProtocol.encodeRedisFrame("value", 1)); if (esmRoundTrip?.payload !== "value" || esmRoundTrip.createdAtMs !== 1) { throw new Error("The packed ESM Redis protocol encoder did not round-trip through the decoder"); } if (redisProtocol.decodeTrackedRedisFrame(redisProtocol.encodeRedisFrame("pending", 0), Buffer.from("0")) !== null) { - throw new Error("The packed ESM Redis protocol encoder did not produce a fenced placeholder frame"); + throw new Error("The packed ESM tracked decoder did not fence an equal timestamp"); } -const esmPlaceholder = redisProtocol.encodeTrackedRedisPlaceholder("pending"); -if ( - esmPlaceholder.frame[0] !== 0 - || esmPlaceholder.nonce.byteLength !== 8 - || redisProtocol.decodeRedisFrame(esmPlaceholder.frame) !== null - || redisProtocol.decodeTrackedRedisFrame(esmPlaceholder.frame, Buffer.from("0")) !== null -) { - throw new Error("The packed ESM tracked placeholder must be unreadable until stamped"); +if (redisProtocol.decodeTrackedRedisFrame(redisProtocol.encodeRedisFrame("value", 1), null)?.payload !== "value") { + throw new Error("The packed ESM tracked decoder did not use zero for a missing watermark"); } if ( "REDIS_FRAME_VERSION" in redisProtocol @@ -826,20 +890,6 @@ if ( ) { throw new Error("The removed wire constants must not be exported by the packed ESM Redis protocol entry"); } -if ( - redisProtocol.resolveTrackedRedisWriteReply(1) !== true - || redisProtocol.resolveTrackedRedisWriteReply(0) !== false -) { - throw new Error("The packed ESM stamp reply resolver did not map replies 0 and 1"); -} -try { - redisProtocol.resolveTrackedRedisWriteReply(2); - throw new Error("Expected a lost-placeholder stamp reply to fail"); -} catch (error) { - if (!(error instanceof root.DialCacheRedisPlaceholderLostError)) { - throw new Error("The lost-placeholder error does not match the root ESM export"); - } -} if (redisProtocol.validateRedisScriptInvalidationReply(1) !== 1) { throw new Error("The packed ESM invalidation reply validator must accept reply 1"); } @@ -866,17 +916,6 @@ for (const invalidCacheTtlMs of [0, 31_536_000_001]) { } } } -// ESM chunk splitting shares one class instance across entries, so also -// prove the brand itself: a hand-branded foreign Error must satisfy the -// root export's Symbol.hasInstance. -const esmBrandedLost = Object.defineProperty( - new Error("lost"), - Symbol.for("dialcache.DialCacheRedisPlaceholderLostError"), - { value: true }, -); -if (!(esmBrandedLost instanceof root.DialCacheRedisPlaceholderLostError)) { - throw new Error("The ESM lost-placeholder brand did not satisfy instanceof"); -} const esmEmptyFrame = Buffer.alloc(10); esmEmptyFrame[0] = 1; esmEmptyFrame.writeBigUInt64BE(1n, 1); @@ -956,6 +995,9 @@ if (inlineCalls !== 1 || inlineSecond !== inlineFirst) { if (!esmRootRuntimeOutput.includes(fallbackTimeoutMarker)) { throw new Error("The packaged ESM only-handle fallback timeout marker is missing"); } + if (!esmRootRuntimeOutput.includes(nodeInvalidationMarker)) { + throw new Error("The packaged ESM node-redis invalidation marker is missing"); + } const { stdout: observerIsolationOutput } = await exec( process.execPath, @@ -1016,7 +1058,7 @@ let payload = Buffer.alloc(4 * 1024 * 1024, 1); const payloadReference = new WeakRef(payload); const redis = { read: async () => ({ payload, createdAtMs: 1 }), - write: async () => true, + write: async () => undefined, invalidate: async () => undefined, }; let resolveTimeout; @@ -1095,19 +1137,25 @@ console.log("${shadowPayloadReleaseMarker}");`, process.execPath, [ "--eval", - `const root = require("dialcache"); + `const { createHash } = require("node:crypto"); +const root = require("dialcache"); const nodeRedis = require("dialcache/node-redis"); require("dialcache/valkey-glide"); require("dialcache/datadog"); const redisProtocol = require("dialcache/redis-protocol"); -// CommonJS bundles duplicate the Lua sources per entry point; a divergence -// forks the protocol (different SHA1s) without failing any behavioral test. -if ( - nodeRedis.dialcacheRedisScripts.dialcacheWriteTrackedStamp.SCRIPT !== redisProtocol.WRITE_TRACKED_STAMP_SCRIPT - || nodeRedis.dialcacheRedisScripts.dialcacheInvalidate.SCRIPT !== redisProtocol.INVALIDATE_CACHE_SCRIPT -) { - throw new Error("The packed CommonJS node-redis Lua sources diverged from the redis-protocol entry"); -} +${packedInvalidationCheckSource} +if (typeof nodeRedis.createNodeRedisDialCacheClient !== "function") { + throw new Error("The packed CommonJS node-redis adapter export is missing"); +} +if ("dialcacheRedisScripts" in nodeRedis) { + throw new Error("The removed CommonJS node-redis script-registration facade is still exported"); +} +const cjsNodeInvalidationCheck = verifyPackedInvalidation({ + createAdapter: (dispatch) => + createPackedNodeRedisInvalidationAdapter(nodeRedis, dispatch, "CommonJS node-redis"), + label: "CommonJS node-redis", + redisProtocol, +}).then(() => console.log("${nodeInvalidationMarker}")); const fallbackTimeoutError = new root.FallbackTimeoutError("PackageRuntime", 1000); if (!(fallbackTimeoutError instanceof root.DialCacheError) || fallbackTimeoutError.timeoutMs !== 1000) { throw new Error("The root CommonJS fallback-timeout error export is invalid"); @@ -1140,22 +1188,8 @@ void (async () => { console.log("${fallbackTimeoutMarker}"); } })(); -try { - nodeRedis.dialcacheRedisScripts.dialcacheWriteTrackedStamp.transformReply(3); - throw new Error("Expected an invalid node-redis script reply to fail"); -} catch (error) { - if (!(error instanceof root.DialCacheRedisProtocolError)) { - throw new Error("The node-redis protocol error does not match the root CommonJS export"); - } -} -if ("MissingKeyConfigError" in root) { - throw new Error("The removed MissingKeyConfigError class must not be exported from the root CommonJS entry"); -} -if ( - "dialcacheRead" in nodeRedis.dialcacheRedisScripts - || "dialcacheReadTracked" in nodeRedis.dialcacheRedisScripts -) { - throw new Error("The removed read scripts must not be registered by the packed CommonJS node-redis entry"); +if ("MissingKeyConfigError" in root || "DialCacheRedisPlaceholderLostError" in root) { + throw new Error("Removed error classes must not be exported from the root CommonJS entry"); } if ( "READ_CACHE_SCRIPT" in redisProtocol @@ -1163,36 +1197,28 @@ if ( ) { throw new Error("The removed read scripts must not be exported by the packed CommonJS Redis protocol entry"); } -if ( - "dialcacheWrite" in nodeRedis.dialcacheRedisScripts - || "dialcacheWriteTracked" in nodeRedis.dialcacheRedisScripts -) { - throw new Error("The removed write scripts must not be registered by the packed CommonJS node-redis entry"); -} if ( "WRITE_CACHE_SCRIPT" in redisProtocol || "WRITE_TRACKED_CACHE_SCRIPT" in redisProtocol ) { throw new Error("The removed write scripts must not be exported by the packed CommonJS Redis protocol entry"); } -if (typeof redisProtocol.WRITE_TRACKED_STAMP_SCRIPT !== "string") { - throw new Error("The packed CommonJS Redis protocol entry must export the tracked stamp script source"); +if ( + "WRITE_TRACKED_STAMP_SCRIPT" in redisProtocol + || "encodeTrackedRedisPlaceholder" in redisProtocol + || "resolveTrackedRedisWriteReply" in redisProtocol +) { + throw new Error("Removed placeholder and stamp helpers must not be exported by the packed CommonJS Redis protocol entry"); } const cjsRoundTrip = redisProtocol.decodeRedisFrame(redisProtocol.encodeRedisFrame("value", 1)); if (cjsRoundTrip?.payload !== "value" || cjsRoundTrip.createdAtMs !== 1) { throw new Error("The packed CommonJS Redis protocol encoder did not round-trip through the decoder"); } if (redisProtocol.decodeTrackedRedisFrame(redisProtocol.encodeRedisFrame("pending", 0), Buffer.from("0")) !== null) { - throw new Error("The packed CommonJS Redis protocol encoder did not produce a fenced placeholder frame"); + throw new Error("The packed CommonJS tracked decoder did not fence an equal timestamp"); } -const cjsPlaceholder = redisProtocol.encodeTrackedRedisPlaceholder("pending"); -if ( - cjsPlaceholder.frame[0] !== 0 - || cjsPlaceholder.nonce.byteLength !== 8 - || redisProtocol.decodeRedisFrame(cjsPlaceholder.frame) !== null - || redisProtocol.decodeTrackedRedisFrame(cjsPlaceholder.frame, Buffer.from("0")) !== null -) { - throw new Error("The packed CommonJS tracked placeholder must be unreadable until stamped"); +if (redisProtocol.decodeTrackedRedisFrame(redisProtocol.encodeRedisFrame("value", 1), null)?.payload !== "value") { + throw new Error("The packed CommonJS tracked decoder did not use zero for a missing watermark"); } if ( "REDIS_FRAME_VERSION" in redisProtocol @@ -1201,20 +1227,6 @@ if ( ) { throw new Error("The removed wire constants must not be exported by the packed CommonJS Redis protocol entry"); } -if ( - redisProtocol.resolveTrackedRedisWriteReply(1) !== true - || redisProtocol.resolveTrackedRedisWriteReply(0) !== false -) { - throw new Error("The packed CommonJS stamp reply resolver did not map replies 0 and 1"); -} -try { - redisProtocol.resolveTrackedRedisWriteReply(2); - throw new Error("Expected a lost-placeholder stamp reply to fail"); -} catch (error) { - if (!(error instanceof root.DialCacheRedisPlaceholderLostError)) { - throw new Error("The lost-placeholder error does not match the root CommonJS export"); - } -} if (redisProtocol.validateRedisScriptInvalidationReply(1) !== 1) { throw new Error("The packed CommonJS invalidation reply validator must accept reply 1"); } @@ -1241,17 +1253,6 @@ for (const invalidCacheTtlMs of [0, 31_536_000_001]) { } } } -// Keep the brand coverage bundler-independent: a hand-branded foreign Error -// must satisfy the root export's Symbol.hasInstance even if CJS ever shares -// chunks the way ESM does. -const cjsBrandedLost = Object.defineProperty( - new Error("lost"), - Symbol.for("dialcache.DialCacheRedisPlaceholderLostError"), - { value: true }, -); -if (!(cjsBrandedLost instanceof root.DialCacheRedisPlaceholderLostError)) { - throw new Error("The CommonJS lost-placeholder brand did not satisfy instanceof"); -} const cjsEmptyFrame = Buffer.alloc(10); cjsEmptyFrame[0] = 1; cjsEmptyFrame.writeBigUInt64BE(1n, 1); @@ -1291,6 +1292,7 @@ if ( throw new Error("The packed CommonJS runtime did not build the disabled() kill-switch overlay"); } void (async () => { + await cjsNodeInvalidationCheck; let calls = 0; const overlayCache = new root.DialCache({ cacheConfigProvider: () => new root.DialCacheKeyConfig({ @@ -1336,6 +1338,9 @@ void (async () => { if (!cjsRootRuntimeOutput.includes(fallbackTimeoutMarker)) { throw new Error("The packaged CommonJS only-handle fallback timeout marker is missing"); } + if (!cjsRootRuntimeOutput.includes(nodeInvalidationMarker)) { + throw new Error("The packaged CommonJS node-redis invalidation marker is missing"); + } await exec( join(workspace, "node_modules", ".bin", "tsc"), ["--project", join(workspace, "tsconfig.root.json")], @@ -1371,7 +1376,7 @@ void (async () => { [ "--input-type=module", "--eval", - `const root = await import("dialcache"); + `const { createHash } = await import("node:crypto"); const glide = await import("dialcache/valkey-glide"); const appGlide = await import("@valkey/valkey-glide"); const otherGlide = await import("dialcache-test-glide"); @@ -1379,9 +1384,12 @@ await import("dialcache/datadog"); await import("dialcache/prometheus"); const redisProtocol = await import("dialcache/redis-protocol"); await import("dialcache/node-redis"); +${packedInvalidationCheckSource} +const esmCreatedAtMs = 1700000000123; if (appGlide.Script === otherGlide.Script) { throw new Error("The package test requires two distinct GLIDE module instances"); } +let esmWriteCommand; const esmFakeGlideClient = { exec: async (batch, _raiseOnError, options) => { if (!(batch instanceof appGlide.Batch) || batch instanceof otherGlide.Batch) { @@ -1390,19 +1398,17 @@ const esmFakeGlideClient = { if (options.decoder !== appGlide.Decoder.Bytes) { throw new Error("The ESM adapter did not use the caller-supplied GLIDE byte decoder"); } - return ["OK", new Error("NOSCRIPT No matching script. Please use EVAL.")]; + return [[esmWriteCommand[2], null]]; }, customCommand: async (args, options) => { - if (args[0] !== "EVAL") { - throw new Error("The ESM adapter's NOSCRIPT recovery must resend the stamp source via EVAL"); - } - if (args[1] !== redisProtocol.WRITE_TRACKED_STAMP_SCRIPT) { - throw new Error("The ESM GLIDE bundle's embedded stamp source diverged from the redis-protocol entry"); + if (args[0] !== "SET") { + throw new Error("The ESM adapter's write must dispatch one native SET"); } if (options.decoder !== appGlide.Decoder.Bytes) { throw new Error("The ESM adapter did not use the caller-supplied GLIDE byte decoder"); } - return 3; + esmWriteCommand = args; + return "OK"; }, }; const esmGlideRuntime = { @@ -1411,47 +1417,39 @@ const esmGlideRuntime = { GlideClusterClient: { [Symbol.hasInstance]: () => false }, }; const adapter = glide.createValkeyGlideDialCacheClient(esmFakeGlideClient, esmGlideRuntime); +const esmNativeDateNow = Date.now; +Date.now = () => esmCreatedAtMs; try { await adapter.write({ valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", cacheTtlMs: 1_000, value: "payload", }); - throw new Error("Expected an invalid GLIDE script reply to fail"); -} catch (error) { - if (!(error instanceof root.DialCacheRedisProtocolError)) { - throw new Error("The GLIDE protocol error does not match the root ESM export", { cause: error }); + if ( + esmWriteCommand[0] !== "SET" + || esmWriteCommand[3] !== "PX" + || esmWriteCommand[4] !== "1000" + || !Buffer.isBuffer(esmWriteCommand[2]) + || esmWriteCommand[2][0] !== 1 + || esmWriteCommand[2].readBigUInt64BE(1) !== BigInt(esmCreatedAtMs) + ) { + throw new Error("The packed ESM GLIDE write did not send one complete client-stamped frame"); } + const trackedRead = await adapter.read({ + valueKey: "tracked:{id}:value", + watermarkKey: "tracked:{id}:watermark", + }); + if (trackedRead?.payload !== "payload" || trackedRead.createdAtMs !== esmCreatedAtMs) { + throw new Error("The packed ESM GLIDE read did not use the caller-supplied Batch runtime"); + } +} finally { + Date.now = esmNativeDateNow; } -const esmInvalidationDispatches = []; -const esmFakeInvalidationClient = { - customCommand: async (args) => { - esmInvalidationDispatches.push(args); - if (esmInvalidationDispatches.length === 1) { - throw new Error("packed invalidation dispatch rejected"); - } - return 1; - }, -}; -const esmInvalidationRuntime = { - ...appGlide, - GlideClient: { [Symbol.hasInstance]: (value) => value === esmFakeInvalidationClient }, - GlideClusterClient: { [Symbol.hasInstance]: () => false }, -}; -await glide - .createValkeyGlideDialCacheClient(esmFakeInvalidationClient, esmInvalidationRuntime) - .invalidate({ watermarkKey: "tracked:{id}:watermark", futureBufferMs: 50 }); -if ( - esmInvalidationDispatches.length !== 2 - || esmInvalidationDispatches[0][0] !== "EVALSHA" - || esmInvalidationDispatches[1][0] !== "EVAL" -) { - throw new Error("The ESM GLIDE invalidation retry did not dispatch EVALSHA then EVAL"); -} -if (esmInvalidationDispatches[1][1] !== redisProtocol.INVALIDATE_CACHE_SCRIPT) { - throw new Error("The ESM GLIDE bundle's embedded invalidation source diverged from the redis-protocol entry"); -}`, +await verifyPackedInvalidation({ + createAdapter: (dispatch) => createPackedGlideInvalidationAdapter(glide, appGlide, dispatch), + label: "ESM GLIDE", + redisProtocol, +});`, ], { cwd: workspace }, ); @@ -1459,7 +1457,7 @@ if (esmInvalidationDispatches[1][1] !== redisProtocol.INVALIDATE_CACHE_SCRIPT) { process.execPath, [ "--eval", - `const root = require("dialcache"); + `const { createHash } = require("node:crypto"); const glide = require("dialcache/valkey-glide"); const appGlide = require("@valkey/valkey-glide"); const otherGlide = require("dialcache-test-glide"); @@ -1467,10 +1465,13 @@ require("dialcache/datadog"); require("dialcache/prometheus"); const redisProtocol = require("dialcache/redis-protocol"); require("dialcache/node-redis"); +${packedInvalidationCheckSource} void (async () => { + const cjsCreatedAtMs = 1700000000123; if (appGlide.Script === otherGlide.Script) { throw new Error("The package test requires two distinct GLIDE module instances"); } + let cjsWriteCommand; const cjsFakeGlideClient = { exec: async (batch, _raiseOnError, options) => { if (!(batch instanceof appGlide.Batch) || batch instanceof otherGlide.Batch) { @@ -1479,19 +1480,17 @@ void (async () => { if (options.decoder !== appGlide.Decoder.Bytes) { throw new Error("The CommonJS adapter did not use the caller-supplied GLIDE byte decoder"); } - return ["OK", new Error("NOSCRIPT No matching script. Please use EVAL.")]; + return [[cjsWriteCommand[2], null]]; }, customCommand: async (args, options) => { - if (args[0] !== "EVAL") { - throw new Error("The CommonJS adapter's NOSCRIPT recovery must resend the stamp source via EVAL"); - } - if (args[1] !== redisProtocol.WRITE_TRACKED_STAMP_SCRIPT) { - throw new Error("The CommonJS GLIDE bundle's embedded stamp source diverged from the redis-protocol entry"); + if (args[0] !== "SET") { + throw new Error("The CommonJS adapter's write must dispatch one native SET"); } if (options.decoder !== appGlide.Decoder.Bytes) { throw new Error("The CommonJS adapter did not use the caller-supplied GLIDE byte decoder"); } - return 3; + cjsWriteCommand = args; + return "OK"; }, }; const cjsGlideRuntime = { @@ -1500,47 +1499,39 @@ void (async () => { GlideClusterClient: { [Symbol.hasInstance]: () => false }, }; const adapter = glide.createValkeyGlideDialCacheClient(cjsFakeGlideClient, cjsGlideRuntime); + const cjsNativeDateNow = Date.now; + Date.now = () => cjsCreatedAtMs; try { await adapter.write({ valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", cacheTtlMs: 1_000, value: "payload", }); - throw new Error("Expected an invalid GLIDE script reply to fail"); - } catch (error) { - if (!(error instanceof root.DialCacheRedisProtocolError)) { - throw new Error("The GLIDE protocol error does not match the root CommonJS export", { cause: error }); + if ( + cjsWriteCommand[0] !== "SET" + || cjsWriteCommand[3] !== "PX" + || cjsWriteCommand[4] !== "1000" + || !Buffer.isBuffer(cjsWriteCommand[2]) + || cjsWriteCommand[2][0] !== 1 + || cjsWriteCommand[2].readBigUInt64BE(1) !== BigInt(cjsCreatedAtMs) + ) { + throw new Error("The packed CommonJS GLIDE write did not send one complete client-stamped frame"); } + const trackedRead = await adapter.read({ + valueKey: "tracked:{id}:value", + watermarkKey: "tracked:{id}:watermark", + }); + if (trackedRead?.payload !== "payload" || trackedRead.createdAtMs !== cjsCreatedAtMs) { + throw new Error("The packed CommonJS GLIDE read did not use the caller-supplied Batch runtime"); + } + } finally { + Date.now = cjsNativeDateNow; } - const cjsInvalidationDispatches = []; - const cjsFakeInvalidationClient = { - customCommand: async (args) => { - cjsInvalidationDispatches.push(args); - if (cjsInvalidationDispatches.length === 1) { - throw new Error("packed invalidation dispatch rejected"); - } - return 1; - }, - }; - const cjsInvalidationRuntime = { - ...appGlide, - GlideClient: { [Symbol.hasInstance]: (value) => value === cjsFakeInvalidationClient }, - GlideClusterClient: { [Symbol.hasInstance]: () => false }, - }; - await glide - .createValkeyGlideDialCacheClient(cjsFakeInvalidationClient, cjsInvalidationRuntime) - .invalidate({ watermarkKey: "tracked:{id}:watermark", futureBufferMs: 50 }); - if ( - cjsInvalidationDispatches.length !== 2 - || cjsInvalidationDispatches[0][0] !== "EVALSHA" - || cjsInvalidationDispatches[1][0] !== "EVAL" - ) { - throw new Error("The CommonJS GLIDE invalidation retry did not dispatch EVALSHA then EVAL"); - } - if (cjsInvalidationDispatches[1][1] !== redisProtocol.INVALIDATE_CACHE_SCRIPT) { - throw new Error("The CommonJS GLIDE bundle's embedded invalidation source diverged from the redis-protocol entry"); - } + await verifyPackedInvalidation({ + createAdapter: (dispatch) => createPackedGlideInvalidationAdapter(glide, appGlide, dispatch), + label: "CommonJS GLIDE", + redisProtocol, + }); })();`, ], { cwd: workspace }, diff --git a/src/datadog.ts b/src/datadog.ts index 1a28fda..b8182e7 100644 --- a/src/datadog.ts +++ b/src/datadog.ts @@ -49,6 +49,7 @@ const METRIC_SUFFIXES = { coalesced: "coalesced.count", shadowValidation: "shadow.count", shadowValueAge: "shadow.value_age", + futureTimestampOffset: "future_timestamp_offset", compression: "compression.count", get: "get.duration", fallback: "fallback.duration", @@ -128,6 +129,10 @@ export class DatadogDialCacheMetrics implements DialCacheMetricsAdapter { this.observe(this.metricNames.shadowValueAge, seconds, shadowValidationTags(labels)); } + observeFutureTimestampOffset(labels: CacheMetricLabels, seconds: number): void { + this.observe(this.metricNames.futureTimestampOffset, seconds, cacheTags(labels)); + } + compression(labels: CompressionMetricLabels): void { this.increment(this.metricNames.compression, { ...cacheTags(labels), outcome: labels.outcome }); } diff --git a/src/dialcache.ts b/src/dialcache.ts index afd8468..9afbf11 100644 --- a/src/dialcache.ts +++ b/src/dialcache.ts @@ -34,7 +34,7 @@ import { } from "./internal/duration.js"; import { LocalCache } from "./internal/local-cache.js"; import { deterministicShadowRampSample } from "./internal/ramp.js"; -import { RedisCache } from "./internal/redis-cache.js"; +import { RedisCache, type FutureFramePolicy } from "./internal/redis-cache.js"; import { fetchKeyConfig, resolveLayerConfigResult, @@ -485,41 +485,44 @@ export class DialCache { * configured, the call rejects rather than reporting an invalidation that * did not occur. * - * This does not synchronously evict local cache hits or untracked Redis values. - * Call it only after the source mutation commits. + * This does not synchronously evict existing local cache hits or untracked + * Redis values. Call it only after the source mutation commits. * - * `futureBufferMs` is an application-owned safety window. When using the - * bundled timestamp protocol, every Redis node eligible for primary promotion - * must have a synchronized system clock. Size the window to cover the maximum - * expected negative clock skew plus source visibility lag and the full - * remaining lifetime of fallback work that may already have observed stale - * data, including serializer dump, Redis client queue and network latency, - * script execution, the write itself, and a safety margin. DialCache does not - * detect or compensate for cross-node clock skew. Violating this assumption - * can suppress tracked cache fills or leave a pre-invalidation value readable - * until it expires or a later invalidation advances the watermark past its - * timestamp. + * The invalidation watermark is the maximum of its prior value and the + * invalidating process's `Date.now() + futureBufferMs`. A tracked read serves + * a complete frame only when its writer timestamp is strictly greater than + * that watermark. A missing watermark is the natural zero baseline. Writes + * never read, create, or extend watermarks; every write is one native SET of + * a complete client-stamped frame. * - * Watermarks are invalidation state and must not be evicted or lost during - * their derived TTL. A missing watermark makes tracked reads miss, but a - * later tracked write initializes a new baseline and cannot recover the lost - * publication fence. Use `noeviction` or an equivalent guarantee when relying - * on that fence, and choose persistence and failover guarantees accordingly; - * DialCache does not issue `WAIT` or provide strong consistency across - * failover. - * There is no universally safe library value. A zero buffer provides no - * stale-publication protection once Redis time advances; an undersized buffer - * may allow stale data to repopulate Redis. An oversized buffer temporarily - * converts more tracked Redis reads into misses and rejects their tracked - * writes, but does not delay or suppress returning fallback values. + * Core caps tracked Redis values at one hour and reports a bounded metric + * error when a dispatched write is clamped. Invalidation keeps a watermark + * for at least two hours, or long enough to outlive its future window plus + * the one-hour value bound and a safety margin; longer and persistent + * existing TTLs are preserved. Watermarks are invalidation state and must not + * be evicted or lost during that interval. Losing one removes its read-time + * invalidation fence and can make an existing frame readable. Use + * `noeviction` or an equivalent guarantee when relying on the fence, and + * choose persistence and failover guarantees accordingly; DialCache does not + * issue `WAIT`. * - * The watermark fences only invocations that reach the tracked Redis write. - * A rejected caller-path write also suppresses the corresponding process-local - * population. Request-local memoization remains unconditional. A ramped-out - * invocation without shadow work does not consult the watermark. A selected - * shadow path for a tracked key consults it for Redis reads and any clean-miss - * fill; untracked shadow work does not. Caller-path request-local and - * process-local publication remains independent. + * `futureBufferMs` is application-owned. Size it through the point where a + * stale SET can become visible: source visibility lag, in-flight fallback and + * serialization work, bounded client queue/reconnect delay, network and Redis + * execution, plus the maximum writer-clock lead over the invalidator. + * DialCache reports future-dated tracked frames through optional metrics but + * does not calibrate clocks. + * A zero buffer fences only frames stamped no later than the invalidation; + * an undersized buffer can admit stale work, while an oversized one causes + * more tracked misses without delaying the returned fallback value. + * + * When an invocation reaches the tracked Redis read/write path, its fallback + * is not published directly to process-local cache; a later validated Redis + * hit may warm it. Local-only, remote-policy-disabled, and ramped-down paths + * retain their local publication policy. Request-local memoization remains + * unconditional, and already-warm local entries are not evicted by this + * remote operation. Ramped-out invocations without shadow work do not consult + * Redis. * * @param futureBufferMs Nonnegative safe integer no greater than * 31,536,000,000 (365 days); defaults to zero for backward compatibility. @@ -784,17 +787,19 @@ export class DialCache { const fallbackLayer = remote.status === "miss" || remoteErrored ? CacheLayer.REMOTE : CacheLayer.LOCAL; const value = await this.callFallback(labelsFor(key, fallbackLayer), fallback); const skipCacheWrite = (remote.status === "miss" || remote.status === "disabled") && remote.skipCacheWrite === true; - let suppressCacheWrite = skipCacheWrite; - if (!suppressCacheWrite && remoteWriteConfig !== undefined) { + // A tracked fallback was not validated against a watermark after the source + // call. If this invocation reached the Redis write path, let a later + // authoritative Redis hit populate local regardless of write success. + const suppressLocalWrite = skipCacheWrite + || (remoteWriteConfig !== undefined && key.trackForInvalidation); + if (!skipCacheWrite && remoteWriteConfig !== undefined) { try { - const wroteRemote = await redisCache.put(key, value, remoteWriteConfig); - suppressCacheWrite = wroteRemote === false; + await redisCache.put(key, value, remoteWriteConfig); } catch (error) { this.logger.warn("Error putting value in Redis cache", error); - suppressCacheWrite = key.trackForInvalidation; } } - if (!suppressCacheWrite && local.status === "miss") { + if (!suppressLocalWrite && local.status === "miss") { await this.putLocalFailOpen(key, value, local.config); } return value; @@ -942,8 +947,8 @@ export class DialCache { operationFinished = true; maybeRelease(); }; - const readShadowFrame = (): Promise => { - const read = redisCache.startPayloadReadForShadow(key, readTimeoutMs); + const readShadowFrame = (futureFramePolicy: FutureFramePolicy): Promise => { + const read = redisCache.startPayloadReadForShadow(key, readTimeoutMs, futureFramePolicy); pendingRedisReads.add(read.settled); void read.settled.then(() => { pendingRedisReads.delete(read.settled); @@ -985,7 +990,7 @@ export class DialCache { if (start.kind === "redis") { let frame: DecodedRedisFrame | null; try { - frame = await readShadowFrame(); + frame = await readShadowFrame("reject"); } catch { return "redis_error"; } @@ -1027,7 +1032,7 @@ export class DialCache { if (shadowFillConfig !== null) { try { - const wroteRemote = await redisCache.putForShadow( + await redisCache.putForShadow( key, sourceValue, shadowFillConfig, @@ -1035,10 +1040,10 @@ export class DialCache { ); // A late result remains the already-emitted whole-job timeout: // dispatch success does not retroactively change its outcome. - if (wroteRemote === null || abandonIfExpired()) { + if (abandonIfExpired()) { return "timeout"; } - return wroteRemote ? "filled" : "fill_blocked"; + return "filled"; } catch (error) { this.logger.warn("Error populating Redis from DialCache shadow work", error); return "fill_error"; @@ -1084,7 +1089,7 @@ export class DialCache { let confirmationFrame: DecodedRedisFrame | null; try { - confirmationFrame = await readShadowFrame(); + confirmationFrame = await readShadowFrame("retain"); } catch { return "confirmation_error"; } @@ -1545,6 +1550,8 @@ function safeMetrics(metrics: DialCacheMetricsAdapter | null): DialCacheMetricsA : {}), observeShadowValueAge: (labels, seconds) => callObserver(() => metrics.observeShadowValueAge?.(labels, seconds)), + observeFutureTimestampOffset: (labels, seconds) => + callObserver(() => metrics.observeFutureTimestampOffset?.(labels, seconds)), observeGet: (labels, seconds) => callObserver(() => metrics.observeGet(labels, seconds)), observeFallback: (labels, seconds) => callObserver(() => metrics.observeFallback(labels, seconds)), observeSerialization: (labels, seconds) => callObserver(() => metrics.observeSerialization(labels, seconds)), @@ -1574,11 +1581,12 @@ function resolveShadowComparator( return comparator ?? isDeepStrictEqual; } -// Frame stamps are epoch-based (Redis server time for tracked writes, writer -// client clock for untracked), so the age uses the epoch clock and clamps -// negative cross-clock skew to zero. A custom client that violates the decode -// contract can hand over a non-finite stamp; recording it would permanently -// poison backend histogram sums, so the observation is skipped instead. +// Frame stamps and the observation both use application-process epoch clocks. +// Core rejects tracked frames that are future-dated when read, but the reader +// clock can step backward before a detached shadow verdict, so clamp that age +// to zero. A custom client that violates the decode contract can hand over a +// non-finite stamp; recording it would permanently poison backend histogram +// sums, so the observation is skipped instead. function shadowValueAgeSeconds(createdAtMs: number): number | undefined { const ageSeconds = (Date.now() - createdAtMs) / 1000; if (!Number.isFinite(ageSeconds)) { diff --git a/src/index.ts b/src/index.ts index cc10a23..6fa2b67 100644 --- a/src/index.ts +++ b/src/index.ts @@ -42,7 +42,6 @@ export type { DialCacheKeyInit } from "./key.js"; export { DialCacheRedisPayloadEncodingError, DialCacheRedisPayloadError, - DialCacheRedisPlaceholderLostError, DialCacheRedisProtocolError, } from "./redis-client.js"; export type { CompressionConfig } from "./internal/compression.js"; diff --git a/src/internal/duration.ts b/src/internal/duration.ts index bd83c86..d22736a 100644 --- a/src/internal/duration.ts +++ b/src/internal/duration.ts @@ -1,6 +1,8 @@ /** Fixed 365-day input ceiling shared by cache TTLs and invalidation buffers. */ export const MAX_SUPPORTED_DURATION_MS = 365 * 24 * 60 * 60 * 1_000; export const MAX_CACHE_TTL_SEC = MAX_SUPPORTED_DURATION_MS / 1_000; +/** Tracked Redis values are bounded so invalidation markers can safely age out. */ +export const MAX_TRACKED_REDIS_VALUE_TTL_MS = 60 * 60 * 1_000; export function isSupportedCacheTtlSec(value: unknown): value is number { return ( @@ -24,8 +26,7 @@ export function cacheTtlSecToMs(ttlSec: number): number { * Validate and ceil an adapter-level write TTL to the protocol's acceptance * domain: fractional milliseconds round up, and the result must be a * positive integer no greater than 365 days. Native SET PX requires an - * integer, and the stamp script re-checks the same domain server-side as - * defense in depth for adapters that skip this guard. + * integer. Core separately caps tracked Redis values at one hour. */ export function ceilSupportedCacheTtlMs(cacheTtlMs: number): number { const ceiled = typeof cacheTtlMs === "number" ? Math.ceil(cacheTtlMs) : Number.NaN; diff --git a/src/internal/redis-cache.ts b/src/internal/redis-cache.ts index ad25fe8..2f48fcb 100644 --- a/src/internal/redis-cache.ts +++ b/src/internal/redis-cache.ts @@ -21,7 +21,7 @@ import { type CompressionConfig, } from "./compression.js"; import { assertValidDeadlineMs, withMonotonicDeadline } from "./deadline.js"; -import { cacheTtlSecToMs } from "./duration.js"; +import { cacheTtlSecToMs, MAX_TRACKED_REDIS_VALUE_TTL_MS } from "./duration.js"; import { fetchKeyConfig, resolveLayerConfigResult, type ResolvedLayerConfig } from "./runtime-config.js"; export interface RedisConfig { @@ -58,6 +58,8 @@ interface StartedRedisRead { readonly settled: Promise; } +export type FutureFramePolicy = "reject" | "retain"; + const defaultSerializer = new JsonSerializer(); const REDIS_FRAME_KEY_SUFFIX = ":dialcache-frame-v1"; const DEFAULT_REMOTE_READ_TIMEOUT_MS = 50; @@ -128,7 +130,7 @@ export class RedisCache { try { let frame: DecodedRedisFrame | null; try { - frame = await this.startPayloadRead(key, readTimeoutMs, false).result; + frame = await this.startPayloadRead(key, readTimeoutMs, metricLayer, false).result; } catch (error) { this.recordError( key, @@ -172,31 +174,33 @@ export class RedisCache { startPayloadReadForShadow( key: DialCacheKey, readTimeoutMs: number, + futureFramePolicy: FutureFramePolicy, ): StartedRedisRead { return this.startMeasuredPayloadRead( key, readTimeoutMs, REMOTE_SHADOW_CACHE_LAYER, true, + futureFramePolicy, ); } - async put(key: DialCacheKey, value: T, config?: { readonly ttlSec: number }): Promise { + async put(key: DialCacheKey, value: T, config?: { readonly ttlSec: number }): Promise { const ttlSec = config?.ttlSec ?? await this.resolveRemoteTtlSec(key); if (ttlSec === null) { - return true; + return; } - return await this.putWithLayer(key, value, ttlSec, CacheLayer.REMOTE); + await this.putWithLayer(key, value, ttlSec, CacheLayer.REMOTE); } - /** Populate a clean detached Redis miss using the caller's resolved policy snapshot. */ + /** Populate a detached Redis miss using the caller's resolved policy snapshot. */ async putForShadow( key: DialCacheKey, value: T, config: { readonly ttlSec: number }, shouldWrite: () => boolean, - ): Promise { - return await this.putWithLayer( + ): Promise { + await this.putWithLayer( key, value, config.ttlSec, @@ -205,27 +209,17 @@ export class RedisCache { ); } - private putWithLayer( - key: DialCacheKey, - value: T, - ttlSec: number, - metricLayer: MetricLayer, - ): Promise; - private putWithLayer( - key: DialCacheKey, - value: T, - ttlSec: number, - metricLayer: MetricLayer, - shouldWrite: () => boolean, - ): Promise; private async putWithLayer( key: DialCacheKey, value: T, ttlSec: number, metricLayer: MetricLayer, shouldWrite?: () => boolean, - ): Promise { - const cacheTtlMs = cacheTtlSecToMs(ttlSec); + ): Promise { + const configuredTtlMs = cacheTtlSecToMs(ttlSec); + const cacheTtlMs = key.trackForInvalidation + ? Math.min(configuredTtlMs, MAX_TRACKED_REDIS_VALUE_TTL_MS) + : configuredTtlMs; const start = performance.now(); let serialized: string | Buffer; @@ -264,21 +258,18 @@ export class RedisCache { } this.recordMetric((metrics) => metrics.observeStoredSize?.(labelsFor(key, metricLayer), payloadSize(serialized))); if (shouldWrite !== undefined && !shouldWrite()) { - return null; + return; + } + if (cacheTtlMs < configuredTtlMs) { + this.recordError(key, metricLayer, "tracked_ttl_clamped"); } try { - const request = { + await this.client.write({ valueKey: this.redisKey(key), cacheTtlMs, value: serialized, - } as const; - return key.trackForInvalidation - ? await this.client.write({ - ...request, - watermarkKey: this.redisWatermarkKeyFromKey(key), - }) - : await this.client.write(request); + }); } catch (error) { this.recordError(key, metricLayer, "cache_write"); throw error; @@ -307,7 +298,9 @@ export class RedisCache { private startPayloadRead( key: DialCacheKey, readTimeoutMs: number, + metricLayer: MetricLayer, unrefTimer: boolean, + futureFramePolicy: FutureFramePolicy = "reject", ): StartedRedisRead { const abortController = new AbortController(); const pending = Promise.resolve().then(() => @@ -319,13 +312,15 @@ export class RedisCache { { timeoutMs: readTimeoutMs, signal: abortController.signal }, ) ); - const result = withMonotonicDeadline({ + const bounded = withMonotonicDeadline({ timeoutMs: readTimeoutMs, operation: () => pending, onTimeout: () => abortController.abort(), timeoutError: () => new RedisReadTimeoutError(key.useCase, readTimeoutMs), unrefTimer, }); + const result = bounded.then((frame) => + this.validateTrackedFrame(key, frame, metricLayer, futureFramePolicy)); return { result, settled: pending.then( @@ -340,10 +335,11 @@ export class RedisCache { readTimeoutMs: number, metricLayer: MetricLayer, unrefTimer: boolean, + futureFramePolicy: FutureFramePolicy, ): StartedRedisRead { const start = performance.now(); this.recordMetric((metrics) => metrics.request(labelsFor(key, metricLayer))); - const read = this.startPayloadRead(key, readTimeoutMs, unrefTimer); + const read = this.startPayloadRead(key, readTimeoutMs, metricLayer, unrefTimer, futureFramePolicy); const result = read.result.then( (frame) => { if (frame === null) { @@ -365,6 +361,33 @@ export class RedisCache { return { result, settled: read.settled }; } + private validateTrackedFrame( + key: DialCacheKey, + frame: DecodedRedisFrame | null, + metricLayer: MetricLayer, + futureFramePolicy: FutureFramePolicy, + ): DecodedRedisFrame | null { + if (frame === null || !key.trackForInvalidation) { + return frame; + } + if (!Number.isSafeInteger(frame.createdAtMs) || frame.createdAtMs < 0) { + return null; + } + + const readerNowMs = Date.now(); + if (frame.createdAtMs > readerNowMs) { + const offsetSeconds = (frame.createdAtMs - readerNowMs) / 1_000; + if (Number.isFinite(offsetSeconds)) { + this.recordMetric((metrics) => metrics.observeFutureTimestampOffset?.( + labelsFor(key, metricLayer), + offsetSeconds, + )); + } + return futureFramePolicy === "reject" ? null : frame; + } + return frame; + } + private async deserializePayload( key: DialCacheKey, payload: RedisCachePayload, diff --git a/src/internal/redis-invalidation.ts b/src/internal/redis-invalidation.ts new file mode 100644 index 0000000..e2dbe1c --- /dev/null +++ b/src/internal/redis-invalidation.ts @@ -0,0 +1,24 @@ +import { createHash } from "node:crypto"; + +import { INVALIDATE_CACHE_SCRIPT } from "./redis-scripts.js"; + +// Redis caches EVAL'd source under sha1(source), so EVALSHA dispatch and EVAL +// recovery must share this exact digest. +export const INVALIDATE_CACHE_SCRIPT_SHA1 = createHash("sha1") + .update(INVALIDATE_CACHE_SCRIPT) + .digest("hex"); + +type RedisInvalidationScriptArguments = readonly [ + futureBufferMs: string, + invalidatedAtMs: string, +]; + +/** Encode the script's ordered ARGV contract; Lua validates both domains. */ +export function buildRedisInvalidationScriptArguments( + futureBufferMs: number, + invalidatedAtMs: number, +): RedisInvalidationScriptArguments { + return [String(futureBufferMs), String(invalidatedAtMs)]; +} + +export { INVALIDATE_CACHE_SCRIPT }; diff --git a/src/internal/redis-payload.ts b/src/internal/redis-payload.ts index fbc3f8e..c857b63 100644 --- a/src/internal/redis-payload.ts +++ b/src/internal/redis-payload.ts @@ -1,5 +1,3 @@ -import { randomBytes } from "node:crypto"; - import { DialCacheRedisPayloadEncodingError, DialCacheRedisPayloadError, @@ -7,15 +5,13 @@ import { type RedisCachePayload, } from "../redis-client.js"; -export const REDIS_FRAME_VERSION = 1; +const REDIS_FRAME_VERSION = 1; const REDIS_ENCODING_UTF8 = 0; const REDIS_ENCODING_BINARY = 1; -/** Version byte of a tracked-write placeholder; no read path serves it. */ -export const REDIS_FRAME_PLACEHOLDER_VERSION = 0; const REDIS_FRAME_TIMESTAMP_OFFSET = 1; -export const REDIS_FRAME_TIMESTAMP_BYTES = 8; +const REDIS_FRAME_TIMESTAMP_BYTES = 8; -export const REDIS_FRAME_HEADER_BYTES = REDIS_FRAME_TIMESTAMP_OFFSET + REDIS_FRAME_TIMESTAMP_BYTES; +const REDIS_FRAME_HEADER_BYTES = REDIS_FRAME_TIMESTAMP_OFFSET + REDIS_FRAME_TIMESTAMP_BYTES; const REDIS_FRAME_MIN_BYTES = REDIS_FRAME_HEADER_BYTES + 1; function validateRedisBulkStringReply(raw: unknown): Buffer | null { @@ -33,21 +29,13 @@ function isSupportedRedisFrame(raw: Buffer | null): raw is Buffer { && raw[0] === REDIS_FRAME_VERSION; } -function parseRedisWatermark(raw: Buffer | null): number | null { - if (raw === null) { - return null; - } +function parseRedisWatermark(raw: Buffer): number | null { const text = raw.toString("utf8"); - const match = /^[0-9]+(?:\.[0-9]+)?/.exec(text); - if (match?.[0].length !== text.length) { + if (!/^[0-9]+$/.test(text)) { return null; } const watermark = Number(text); - return Number.isFinite(watermark) ? watermark : null; -} - -function redisPayloadEncoding(value: RedisCachePayload): number { - return Buffer.isBuffer(value) ? REDIS_ENCODING_BINARY : REDIS_ENCODING_UTF8; + return watermark <= Number.MAX_SAFE_INTEGER ? watermark : null; } function decodeRedisPayload(raw: Buffer): RedisCachePayload { @@ -62,68 +50,46 @@ function decodeRedisPayload(raw: Buffer): RedisCachePayload { throw new DialCacheRedisPayloadEncodingError("Invalid DialCache Redis payload encoding"); } -function encodeFrameBytes(payload: RedisCachePayload, version: number, stampBytes: Buffer): Buffer { - const payloadBytes = Buffer.isBuffer(payload) ? payload.length : Buffer.byteLength(payload, "utf8"); - const frame = Buffer.allocUnsafe(REDIS_FRAME_MIN_BYTES + payloadBytes); - frame[0] = version; - stampBytes.copy(frame, REDIS_FRAME_TIMESTAMP_OFFSET); - frame[REDIS_FRAME_HEADER_BYTES] = redisPayloadEncoding(payload); - if (Buffer.isBuffer(payload)) { - payload.copy(frame, REDIS_FRAME_MIN_BYTES); - } else { - frame.write(payload, REDIS_FRAME_MIN_BYTES, "utf8"); - } - return frame; -} - /** * Encode a serializer payload into a servable DialCache Redis frame. * - * Untracked writes stamp a client-clock `createdAtMs`. Untracked reads never - * consult the stamp for serving or miss decisions, but they surface it as the - * decoded frame's `createdAtMs`, which feeds the shadow value-age - * observation — so stamp real client time, not a constant. Tracked writes - * must not use this directly — they pair `encodeTrackedRedisPlaceholder` - * with `WRITE_TRACKED_STAMP_SCRIPT` instead. + * Writes stamp a client-clock `createdAtMs`. Core rejects tracked frames dated + * after the reader's clock and uses decoded timestamps for shadow value-age + * observations, so stamp real client time, not a constant. */ export function encodeRedisFrame(payload: RedisCachePayload, createdAtMs: number): Buffer { - if (!Number.isSafeInteger(createdAtMs) || createdAtMs < 0) { + if (!isValidRedisTimestampMs(createdAtMs)) { throw new RangeError("DialCache frame createdAtMs must be a nonnegative safe integer"); } - const timestamp = Buffer.allocUnsafe(REDIS_FRAME_TIMESTAMP_BYTES); - timestamp.writeBigUInt64BE(BigInt(createdAtMs)); - return encodeFrameBytes(payload, REDIS_FRAME_VERSION, timestamp); + const isBinary = Buffer.isBuffer(payload); + const payloadBytes = isBinary ? payload.length : Buffer.byteLength(payload, "utf8"); + const frame = Buffer.allocUnsafe(REDIS_FRAME_MIN_BYTES + payloadBytes); + frame[0] = REDIS_FRAME_VERSION; + frame.writeBigUInt64BE(BigInt(createdAtMs), REDIS_FRAME_TIMESTAMP_OFFSET); + frame[REDIS_FRAME_HEADER_BYTES] = isBinary ? REDIS_ENCODING_BINARY : REDIS_ENCODING_UTF8; + if (isBinary) { + payload.copy(frame, REDIS_FRAME_MIN_BYTES); + } else { + frame.write(payload, REDIS_FRAME_MIN_BYTES, "utf8"); + } + return frame; } -export interface TrackedRedisPlaceholder { - /** Version-0 frame that no read path serves until the stamp promotes it. */ - readonly frame: Buffer; - /** Per-write identity passed to `WRITE_TRACKED_STAMP_SCRIPT` as its nonce argument. */ - readonly nonce: Buffer; +/** Validate an application-clock epoch timestamp before mutation dispatch. */ +export function assertValidRedisTimestampMs(timestampMs: number): void { + if (!isValidRedisTimestampMs(timestampMs)) { + throw new RangeError("DialCache Redis timestamp must be a nonnegative safe integer"); + } } -/** - * Encode the placeholder frame a tracked write pairs with - * `WRITE_TRACKED_STAMP_SCRIPT`. - * - * The frame carries the placeholder version byte, so both read paths treat it - * as a miss, and a fresh random nonce where a stamped frame carries its - * timestamp. The stamp promotes the frame — patching version and server-time - * timestamp — only when the stored header matches this exact nonce, so it can - * never publish a placeholder left behind by a different write. Mint one - * placeholder per logical write: client-level retries must reuse the same - * frame and nonce so a retried SET re-establishes the placeholder its stamp - * expects. - */ -export function encodeTrackedRedisPlaceholder(payload: RedisCachePayload): TrackedRedisPlaceholder { - const nonce = randomBytes(REDIS_FRAME_TIMESTAMP_BYTES); - return { frame: encodeFrameBytes(payload, REDIS_FRAME_PLACEHOLDER_VERSION, nonce), nonce }; +function isValidRedisTimestampMs(timestampMs: number): boolean { + return Number.isSafeInteger(timestampMs) && timestampMs >= 0; } /** * Decode an untracked DialCache frame returned as a Redis bulk string into - * its serializer payload and header creation time (the writer's informational - * client clock). Missing, short, and unsupported-version frames are cache + * its serializer payload and header creation time (the writer's application + * clock). Missing, short, and unsupported-version frames are cache * misses. Invalid runtime reply types and unsupported payload encodings throw * typed errors. */ @@ -141,9 +107,10 @@ export function decodeRedisFrame(raw: unknown): DecodedRedisFrame | null { /** * Decode a tracked DialCache frame against a watermark from the same atomic, * authoritative snapshot into its serializer payload and header creation time - * (Redis server time written by the stamp script). Missing or malformed state - * and frames created at or before the watermark are cache misses. Invalid - * runtime reply types and unsupported payload encodings throw typed errors. + * (application time supplied by the writer). A missing watermark is the + * natural zero baseline; malformed state and frames created at or before the + * watermark are cache misses. Invalid runtime reply types and unsupported + * payload encodings throw typed errors. */ export function decodeTrackedRedisFrame( raw: unknown, @@ -154,7 +121,7 @@ export function decodeTrackedRedisFrame( if (!isSupportedRedisFrame(frame)) { return null; } - const watermark = parseRedisWatermark(watermarkFrame); + const watermark = watermarkFrame === null ? 0 : parseRedisWatermark(watermarkFrame); if (watermark === null) { return null; } diff --git a/src/internal/redis-script-reply.ts b/src/internal/redis-script-reply.ts index b018aab..fa718ae 100644 --- a/src/internal/redis-script-reply.ts +++ b/src/internal/redis-script-reply.ts @@ -1,7 +1,4 @@ -import { - DialCacheRedisPlaceholderLostError, - DialCacheRedisProtocolError, -} from "../redis-client.js"; +import { DialCacheRedisProtocolError } from "../redis-client.js"; export function validateRedisSetReply(reply: unknown): void { const text = typeof reply === "string" @@ -14,28 +11,6 @@ export function validateRedisSetReply(reply: unknown): void { } } -export function validateRedisScriptWriteReply(reply: unknown): 0 | 1 | 2 { - if (reply !== 0 && reply !== 1 && reply !== 2) { - throw new DialCacheRedisProtocolError("Invalid DialCache Redis write reply; expected integer 0, 1, or 2"); - } - return reply; -} - -/** - * Map a validated stamp reply onto the write() boolean contract: 0 (fenced) - * is false, 1 (stamped) is true, and 2 — the paired placeholder was gone — - * fails the write so split pairs surface through the normal fail-open path. - */ -export function resolveTrackedRedisWriteReply(reply: unknown): boolean { - const stamp = validateRedisScriptWriteReply(reply); - if (stamp === 2) { - throw new DialCacheRedisPlaceholderLostError( - "DialCache tracked write lost its placeholder before the stamp; the SET was rejected, overwritten, or expired", - ); - } - return stamp === 1; -} - export function validateRedisScriptInvalidationReply(reply: unknown): 1 { if (reply !== 1) { throw new DialCacheRedisProtocolError("Invalid DialCache Redis invalidate reply; expected integer 1"); diff --git a/src/internal/redis-scripts.ts b/src/internal/redis-scripts.ts index 1fcfcd7..7020c0e 100644 --- a/src/internal/redis-scripts.ts +++ b/src/internal/redis-scripts.ts @@ -1,116 +1,68 @@ -import { MAX_SUPPORTED_DURATION_MS } from "./duration.js"; import { - REDIS_FRAME_HEADER_BYTES, - REDIS_FRAME_PLACEHOLDER_VERSION, - REDIS_FRAME_TIMESTAMP_BYTES, - REDIS_FRAME_VERSION, -} from "./redis-payload.js"; + MAX_SUPPORTED_DURATION_MS, + MAX_TRACKED_REDIS_VALUE_TTL_MS, +} from "./duration.js"; const WATERMARK_TTL_MARGIN_MS = 60_000; -const PARSE_WATERMARK_LUA = String.raw`local function parse_watermark(raw) - if not string.match(raw, "^%d+$") and not string.match(raw, "^%d+%.%d+$") then +/** + * Current protocol floor for invalidation-owned watermarks. + * + * This relationship protects one release's values; it is not a rolling-version + * compatibility guarantee. Raising the tracked-value cap requires a gated + * protocol cutover because already-deployed invalidators retain their older + * compiled floor. + */ +export const MIN_WATERMARK_TTL_MS = 2 * MAX_TRACKED_REDIS_VALUE_TTL_MS; + +const PARSE_SAFE_INTEGER_LUA = String.raw`local function parse_safe_integer(raw) + if not string.match(raw, "^%d+$") then return nil end local value = tonumber(raw) - if not value or value >= math.huge then + if not value or value > ${Number.MAX_SAFE_INTEGER} then return nil end return value end`; -const CEIL_FINITE_NUMBER_LUA = String.raw`local function ceil_finite_number(raw) - local value = tonumber(raw) - if not value or value ~= value or value >= math.huge or value <= -math.huge then - return nil - end - return math.ceil(value) -end`; - -const VALIDATE_STAMP_ARGUMENTS_LUA = String.raw`local cache_ttl_ms = ceil_finite_number(ARGV[1]) -if not cache_ttl_ms or cache_ttl_ms <= 0 or cache_ttl_ms > ${MAX_SUPPORTED_DURATION_MS} then - return redis.error_reply("ERR invalid DialCache TTL") -end -if string.len(ARGV[2]) ~= ${REDIS_FRAME_TIMESTAMP_BYTES} then - return redis.error_reply("ERR invalid DialCache stamp nonce") -end`; - -const REDIS_TIME_LUA = String.raw`local redis_time = redis.call("TIME") -local now_ms = tonumber(redis_time[1]) * 1000 + math.floor(tonumber(redis_time[2]) / 1000)`; - -export const WRITE_TRACKED_STAMP_SCRIPT = [ - PARSE_WATERMARK_LUA, - CEIL_FINITE_NUMBER_LUA, - VALIDATE_STAMP_ARGUMENTS_LUA, - REDIS_TIME_LUA, - String.raw`local raw_watermark = redis.call("GET", KEYS[2]) -local watermark = 0 -if raw_watermark then - watermark = parse_watermark(raw_watermark) - if not watermark then - return redis.error_reply("ERR invalid DialCache watermark") - end -end - -if watermark >= now_ms then - -- A fenced fallback write removes the placeholder it paired with, along with any - -- stale frame that led to it. The UNLINK stays unconditional: any frame present - -- here is already fenced, and removing a foreign in-flight placeholder only - -- forces that writer's honest reply-2 failure. Reads that fail before reaching - -- this script cannot benefit from this partial mitigation. - redis.call("UNLINK", KEYS[1]) - return 0 -end`, - String.raw`local stamped = 1 -if redis.call("GETRANGE", KEYS[1], 0, ${REDIS_FRAME_HEADER_BYTES - 1}) == string.char(${REDIS_FRAME_PLACEHOLDER_VERSION}) .. ARGV[2] then - redis.call("SETRANGE", KEYS[1], 0, string.char(${REDIS_FRAME_VERSION}) .. struct.pack(">I8", now_ms)) -else - -- The placeholder this stamp paired with is gone: its SET was rejected, - -- overwritten, or expired. Promoting any other frame could publish a value - -- this write does not own, so leave the key untouched and report 2. - stamped = 2 -end`, - String.raw`local desired_ttl_ms = cache_ttl_ms + ${WATERMARK_TTL_MARGIN_MS} -if not raw_watermark then - redis.call("SET", KEYS[2], "0", "PX", desired_ttl_ms) -else - local current_ttl_ms = redis.call("PTTL", KEYS[2]) - if current_ttl_ms == -2 then - redis.call("SET", KEYS[2], raw_watermark, "PX", desired_ttl_ms) - elseif current_ttl_ms ~= -1 and current_ttl_ms < desired_ttl_ms then - redis.call("PEXPIRE", KEYS[2], desired_ttl_ms) - end -end`, - "return stamped", -].join("\n\n"); - export const INVALIDATE_CACHE_SCRIPT = [ - PARSE_WATERMARK_LUA, - CEIL_FINITE_NUMBER_LUA, - String.raw`local future_buffer_ms = ceil_finite_number(ARGV[1]) + PARSE_SAFE_INTEGER_LUA, + String.raw`local future_buffer_ms = parse_safe_integer(ARGV[1]) if not future_buffer_ms or future_buffer_ms < 0 or future_buffer_ms > ${MAX_SUPPORTED_DURATION_MS} then return redis.error_reply("ERR invalid DialCache future buffer") +end +local invalidated_at_ms = parse_safe_integer(ARGV[2]) +if not invalidated_at_ms or invalidated_at_ms > ${Number.MAX_SAFE_INTEGER} - future_buffer_ms then + return redis.error_reply("ERR invalid DialCache invalidatedAtMs") end`, - REDIS_TIME_LUA, - String.raw`local proposed_watermark = now_ms + future_buffer_ms -local raw_watermark = redis.call("GET", KEYS[1]) + String.raw`local proposed_watermark = invalidated_at_ms + future_buffer_ms +local raw_watermark = redis.pcall("GET", KEYS[1]) +if type(raw_watermark) == "table" and raw_watermark.err then + if not string.match(raw_watermark.err, "^WRONGTYPE ") then + return raw_watermark + end + -- A wrong-type key cannot contain a valid watermark. Treat it as absent so + -- the final SET repairs it, while preserving every other Redis error. + raw_watermark = false +end local current_watermark = 0 if raw_watermark then - local parsed_watermark = parse_watermark(raw_watermark) + local parsed_watermark = parse_safe_integer(raw_watermark) if parsed_watermark then current_watermark = parsed_watermark end end -local watermark = math.ceil(math.max(current_watermark, proposed_watermark)) +local watermark = math.max(current_watermark, proposed_watermark) local current_ttl_ms = -2 if raw_watermark then current_ttl_ms = redis.call("PTTL", KEYS[1]) end local desired_ttl_ms = math.max( - future_buffer_ms + ${WATERMARK_TTL_MARGIN_MS}, - watermark - now_ms + ${WATERMARK_TTL_MARGIN_MS} + ${MIN_WATERMARK_TTL_MS}, + watermark - invalidated_at_ms + ${MAX_TRACKED_REDIS_VALUE_TTL_MS} + ${WATERMARK_TTL_MARGIN_MS} ) if current_ttl_ms > desired_ttl_ms then desired_ttl_ms = current_ttl_ms diff --git a/src/metrics.ts b/src/metrics.ts index 998469f..2bda225 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -16,7 +16,6 @@ export type ShadowValidationOutcome = | "mismatch" | "superseded" | "filled" - | "fill_blocked" | "fill_error" | "redis_error" | "source_error" @@ -43,13 +42,14 @@ export type CompressionOutcome = | "read_over_limit"; /** Bounded reasons for skipping cache work; policy_disabled means a shared layer has no effective TTL. */ export type DisabledReason = "context" | "policy_disabled" | "invalid_ttl" | "invalid_ramp" | "ramped_down" | "config_error"; -/** Stable failure sites used instead of backend- or application-defined error names. */ +/** Stable failure sites and configuration signals; never backend- or application-defined names. */ export type MetricErrorKind = | "key_construction" | "config_resolution" | "cache_read" | "cache_read_timeout" | "cache_write" + | "tracked_ttl_clamped" | "serialization_load" | "serialization_dump" | "compression" @@ -121,13 +121,20 @@ export interface DialCacheMetricsAdapter { * a mismatch, after the confirming re-read): the observing process's epoch * clock minus the validated frame's `createdAtMs`, clamped at zero. * Emitted only alongside terminal `match` and `mismatch` outcomes; other - * outcomes deliver no verdict on a retained value. Tracked frames are - * stamped with Redis server time and untracked frames with the writer's - * client clock, so the age mixes clocks and is coarse operational - * evidence, not a precise measurement. Optional so existing custom - * adapters keep compiling without changes. + * outcomes deliver no verdict on a retained value. Frames are stamped with + * the writer application's epoch clock, so cross-process skew still makes + * this coarse operational evidence rather than a precise measurement. + * Optional so existing custom adapters keep compiling without changes. */ observeShadowValueAge?(labels: ShadowValidationMetricLabels, seconds: number): void; + /** + * Positive offset in seconds when a decoded tracked Redis frame is dated + * after the observing process's epoch clock. Serving and initial shadow reads + * fail closed as misses; a shadow confirmation retains the frame solely for + * payload comparison. This is a workload-shaped diagnostic, not proof of + * clock skew. Optional so existing custom adapters keep compiling. + */ + observeFutureTimestampOffset?(labels: CacheMetricLabels, seconds: number): void; // Optional so existing custom adapters keep compiling without changes. compression?(labels: CompressionMetricLabels): void; observeGet(labels: CacheMetricLabels, seconds: number): void; diff --git a/src/node-redis.ts b/src/node-redis.ts index 4c80358..c7f28d2 100644 --- a/src/node-redis.ts +++ b/src/node-redis.ts @@ -1,25 +1,23 @@ -import { commandOptions, defineScript } from "redis"; +import { commandOptions } from "redis"; import { + buildRedisInvalidationScriptArguments, INVALIDATE_CACHE_SCRIPT, - WRITE_TRACKED_STAMP_SCRIPT, -} from "./internal/redis-scripts.js"; + INVALIDATE_CACHE_SCRIPT_SHA1, +} from "./internal/redis-invalidation.js"; import { + assertValidRedisTimestampMs, decodeRedisFrame, decodeTrackedRedisFrame, encodeRedisFrame, - encodeTrackedRedisPlaceholder, } from "./internal/redis-payload.js"; import { ceilSupportedCacheTtlMs } from "./internal/duration.js"; import { - resolveTrackedRedisWriteReply, validateRedisScriptInvalidationReply, - validateRedisScriptWriteReply, validateRedisSetReply, } from "./internal/redis-script-reply.js"; import { DialCacheRedisPayloadError, - DialCacheRedisProtocolError, type DialCacheRedisClient, } from "./redis-client.js"; @@ -31,87 +29,9 @@ type BufferReplyOptions = ReturnType< >; // Redis bulk strings are binary data; decoding them as UTF-8 would corrupt arbitrary serializer output. const bufferReplyOptions: BufferReplyOptions = commandOptions({ returnBuffers: true }); -const writeReply = (reply: number): number => validateRedisScriptWriteReply(reply); -const invalidationReply = (reply: number): number => validateRedisScriptInvalidationReply(reply); type NodeRedisArgument = string | Buffer; -interface NodeRedisScript, Reply> { - readonly SCRIPT: string; - readonly SHA1: string; - readonly NUMBER_OF_KEYS: number; - readonly FIRST_KEY_INDEX: number; - readonly IS_READ_ONLY: boolean; - transformArguments(...args: Args): Array; - transformReply(reply: Reply): Reply; -} - -type NodeRedisScriptConfig, Reply> = Omit, "SHA1">; - -function defineDialCacheScript, Reply>( - config: NodeRedisScriptConfig, -): NodeRedisScript { - return defineScript(config); -} - -/** - * DialCache's client wiring, not a write API: the registered methods return - * raw script replies. `dialcacheWriteTrackedStamp` replies `0 | 1 | 2`, and - * `2` means the placeholder was lost — not success. Code invoking these - * methods directly must map stamp replies through - * `resolveTrackedRedisWriteReply` from `dialcache/redis-protocol`, which - * throws `DialCacheRedisPlaceholderLostError` on `2`. - */ -export type DialCacheNodeRedisScripts = { - readonly dialcacheWriteTrackedStamp: NodeRedisScript< - [valueKey: string, watermarkKey: string, cacheTtlMs: number, nonce: Buffer], - number - >; - readonly dialcacheInvalidate: NodeRedisScript< - [watermarkKey: string, futureBufferMs: number], - number - >; -}; - -/** See {@link DialCacheNodeRedisScripts}: wiring for the adapter, not a direct write API. */ -export const dialcacheRedisScripts: DialCacheNodeRedisScripts = { - dialcacheWriteTrackedStamp: defineDialCacheScript({ - SCRIPT: WRITE_TRACKED_STAMP_SCRIPT, - NUMBER_OF_KEYS: 2, - FIRST_KEY_INDEX: 0, - IS_READ_ONLY: false, - transformArguments( - valueKey: string, - watermarkKey: string, - cacheTtlMs: number, - nonce: Buffer, - ): Array { - return [valueKey, watermarkKey, String(cacheTtlMs), nonce]; - }, - transformReply: writeReply, - }), - dialcacheInvalidate: defineDialCacheScript({ - SCRIPT: INVALIDATE_CACHE_SCRIPT, - NUMBER_OF_KEYS: 1, - FIRST_KEY_INDEX: 0, - IS_READ_ONLY: false, - transformArguments(watermarkKey: string, futureBufferMs: number): Array { - return [watermarkKey, String(futureBufferMs)]; - }, - transformReply: invalidationReply, - }), -}; - -interface NodeRedisWriteClient { - dialcacheWriteTrackedStamp( - valueKey: string, - watermarkKey: string, - cacheTtlMs: number, - nonce: Buffer, - ): Promise; - dialcacheInvalidate(watermarkKey: string, futureBufferMs: number): Promise; -} - -interface NodeRedisStandaloneClient extends NodeRedisWriteClient { +interface NodeRedisStandaloneClient { get(options: BufferReplyOptions, valueKey: string): Promise; sendCommand( args: Array, @@ -119,7 +39,7 @@ interface NodeRedisStandaloneClient extends NodeRedisWriteClient { ): Promise; } -interface NodeRedisClusterClient extends NodeRedisWriteClient { +interface NodeRedisClusterClient { /** Public node-redis Cluster topology view, used only to distinguish its sendCommand overload. */ readonly masters: ReadonlyArray; get(options: BufferReplyOptions, valueKey: string): Promise; @@ -191,13 +111,12 @@ function sendFrameSet( * Create a resource-free semantic view over a caller-owned node-redis client. * Read signals are passed to node-redis so queued commands can be removed when * supported. Aborting after dispatch does not unsend a command or prove the - * server stopped executing it. Tracked writes enqueue their placeholder SET - * and stamp script in one synchronous tick, so node-redis pipelines them in - * order on one connection (per slot node in cluster mode). Invalidation - * retries any dispatch rejection other than a reply-domain violation once by + * server stopped executing it. Every write is one native SET of a complete + * client-stamped frame. Invalidation retries any EVALSHA rejection once by * re-sending the script source as EVAL — the script is idempotent, so a - * duplicate run is harmless — and a failed retry surfaces unmodified, with - * the original rejection discarded. node-redis has no per-command deadline: + * duplicate run is harmless — and a failed retry surfaces unmodified. An + * accepted reply is validated after dispatch and is never retried. node-redis + * has no per-command deadline: * `disableOfflineQueue`, `commandsQueueMaxLength`, and `reconnectStrategy` * bound queueing and dispatch, not the reply wait, so with the offline queue * enabled a retry issued during a disconnect can wait until reconnect. The @@ -205,14 +124,6 @@ function sendFrameSet( * work, and closing the client. */ export function createNodeRedisDialCacheClient(client: NodeRedisClient): DialCacheRedisClient { - if ( - typeof client.dialcacheWriteTrackedStamp !== "function" - || typeof client.dialcacheInvalidate !== "function" - ) { - throw new TypeError( - "node-redis DialCache requires a client created with scripts: dialcacheRedisScripts", - ); - } return { async read({ valueKey, watermarkKey }, context) { const options: BufferReplyOptions = context === undefined @@ -230,49 +141,33 @@ export function createNodeRedisDialCacheClient(client: NodeRedisClient): DialCac return decodeTrackedRedisFrame(rawValue, rawWatermark); }, async write(request) { - const { valueKey, watermarkKey, value } = request; + const { valueKey, value } = request; const cacheTtlMs = ceilSupportedCacheTtlMs(request.cacheTtlMs); - if (watermarkKey === undefined) { - validateRedisSetReply( - await sendFrameSet(client, valueKey, encodeRedisFrame(value, Date.now()), cacheTtlMs), - ); - return true; - } - const { frame, nonce } = encodeTrackedRedisPlaceholder(value); - // Both commands must enqueue in this synchronous tick so they pipeline - // in order; an await between them would allow reordering around them. - const setPromise = sendFrameSet(client, valueKey, frame, cacheTtlMs); - // Observe the SET unconditionally so a synchronous throw before - // allSettled cannot leave its rejection unhandled. - setPromise.catch(() => undefined); - const stampPromise = client.dialcacheWriteTrackedStamp(valueKey, watermarkKey, cacheTtlMs, nonce); - const [setResult, stampResult] = await Promise.allSettled([setPromise, stampPromise]); - // A failed SET is the write outcome even when the stamp settled. - if (setResult.status === "rejected") { - throw setResult.reason; - } - validateRedisSetReply(setResult.value); - if (stampResult.status === "rejected") { - throw stampResult.reason; - } - return resolveTrackedRedisWriteReply(stampResult.value); + validateRedisSetReply( + await sendFrameSet(client, valueKey, encodeRedisFrame(value, Date.now()), cacheTtlMs), + ); }, async invalidate({ watermarkKey, futureBufferMs }) { + const invalidatedAtMs = Date.now(); + assertValidRedisTimestampMs(invalidatedAtMs); + const invalidateArgs = buildRedisInvalidationScriptArguments( + futureBufferMs, + invalidatedAtMs, + ); let raw: unknown; try { - raw = await client.dialcacheInvalidate(watermarkKey, futureBufferMs); - } catch (error) { - // The registered transformReply validates inside the returned - // promise, so a reply-domain violation surfaces here as a rejection; - // it is deterministic and must not be retried. Any other rejection - // is retried once with the source: the invalidation script is + raw = await sendKeyedCommand( + client, + watermarkKey, + ["EVALSHA", INVALIDATE_CACHE_SCRIPT_SHA1, "1", watermarkKey, ...invalidateArgs], + bufferReplyOptions, + ); + } catch { + // Any rejection is retried once with the source: the invalidation script is // idempotent (the watermark only advances and its TTL only widens), // so a duplicate run after an ambiguous failure is harmless, and // EVAL self-heals both a flushed script cache and an // EVALSHA-rejecting proxy without depending on error wording. - if (error instanceof DialCacheRedisProtocolError) { - throw error; - } // A failed retry surfaces unmodified, discarding this original // rejection: node-redis rejects every command flushed by a single // disconnect with one shared error instance — the same object its @@ -281,7 +176,13 @@ export function createNodeRedisDialCacheClient(client: NodeRedisClient): DialCac raw = await sendKeyedCommand( client, watermarkKey, - ["EVAL", INVALIDATE_CACHE_SCRIPT, "1", watermarkKey, String(futureBufferMs)], + [ + "EVAL", + INVALIDATE_CACHE_SCRIPT, + "1", + watermarkKey, + ...invalidateArgs, + ], bufferReplyOptions, ); } diff --git a/src/prometheus.ts b/src/prometheus.ts index 7674675..e3ee92e 100644 --- a/src/prometheus.ts +++ b/src/prometheus.ts @@ -60,6 +60,25 @@ const SIZE_BUCKETS = [100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000]; const RATIO_BUCKETS = [0.05, 0.1, 0.2, 0.3, 0.5, 0.7, 0.9, 1]; // Value ages span seconds to the 365-day TTL ceiling: 1s..15m, then 1h, 3h, 12h, 1d, 3d, 7d. const VALUE_AGE_BUCKETS = [1, 5, 15, 60, 300, 900, 3_600, 10_800, 43_200, 86_400, 259_200, 604_800]; +const FUTURE_TIMESTAMP_OFFSET_BUCKETS = [ + 0.001, + 0.005, + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1, + 5, + 15, + 60, + 300, + 900, + 3_600, + 10_800, + 43_200, +]; export class PrometheusDialCacheMetrics implements DialCacheMetricsAdapter { private readonly requestCounter: Counter; @@ -70,6 +89,7 @@ export class PrometheusDialCacheMetrics implements DialCacheMetricsAdapter { private readonly coalescedCounter: Counter; private readonly shadowValidationCounter: Counter; private readonly shadowValueAgeHistogram: Histogram; + private readonly futureTimestampOffsetHistogram: Histogram; private readonly compressionCounter: Counter; private readonly getTimer: Histogram; private readonly fallbackTimer: Histogram; @@ -93,6 +113,7 @@ export class PrometheusDialCacheMetrics implements DialCacheMetricsAdapter { this.coalescedCounter = counter(registry, collectors.coalescedCounter); this.shadowValidationCounter = counter(registry, collectors.shadowValidationCounter); this.shadowValueAgeHistogram = histogram(registry, collectors.shadowValueAgeHistogram); + this.futureTimestampOffsetHistogram = histogram(registry, collectors.futureTimestampOffsetHistogram); this.compressionCounter = counter(registry, collectors.compressionCounter); this.getTimer = histogram(registry, collectors.getTimer); this.fallbackTimer = histogram(registry, collectors.fallbackTimer); @@ -148,6 +169,13 @@ export class PrometheusDialCacheMetrics implements DialCacheMetricsAdapter { this.shadowValueAgeHistogram.observe(shadowValidationLabels(labels), seconds); } + observeFutureTimestampOffset(labels: CacheMetricLabels, seconds: number): void { + if (!Number.isFinite(seconds) || seconds <= 0) { + return; + } + this.futureTimestampOffsetHistogram.observe(cacheLabels(labels), seconds); + } + compression(labels: CompressionMetricLabels): void { this.compressionCounter.inc({ ...cacheLabels(labels), outcome: labels.outcome }); } @@ -254,6 +282,13 @@ function collectorConfigs(prefix: string) { labelNames: ["cache_namespace", "use_case", "key_type", "outcome"], buckets: VALUE_AGE_BUCKETS, }, + futureTimestampOffsetHistogram: { + type: "histogram", + name: `${prefix}dialcache_future_timestamp_offset_histogram`, + help: "Positive offset in seconds of tracked Redis frames dated after the observing DialCache process clock.", + labelNames: ["cache_namespace", "use_case", "key_type", "layer"], + buckets: FUTURE_TIMESTAMP_OFFSET_BUCKETS, + }, compressionCounter: { type: "counter", name: `${prefix}dialcache_compression_counter`, diff --git a/src/redis-client.ts b/src/redis-client.ts index 1b58f3d..aa934b1 100644 --- a/src/redis-client.ts +++ b/src/redis-client.ts @@ -3,7 +3,6 @@ import type { Awaitable } from "./config.js"; const redisPayloadErrorBrand = Symbol.for("dialcache.DialCacheRedisPayloadError"); const redisPayloadEncodingErrorBrand = Symbol.for("dialcache.DialCacheRedisPayloadEncodingError"); const redisProtocolErrorBrand = Symbol.for("dialcache.DialCacheRedisProtocolError"); -const redisPlaceholderLostErrorBrand = Symbol.for("dialcache.DialCacheRedisPlaceholderLostError"); export class DialCacheRedisPayloadError extends Error { static [Symbol.hasInstance](value: unknown): boolean { @@ -59,31 +58,6 @@ export class DialCacheRedisProtocolError extends Error { } } -/** - * A tracked write's stamp found no placeholder carrying its nonce: the paired - * SET was rejected, overwritten by a concurrent writer, expired, or removed - * by a fenced write. The value was not published, and DialCache suppresses the - * corresponding process-local publication. Same-key write contention produces - * a benign floor of these, concentrated on hot keys at TTL expiry. - */ -export class DialCacheRedisPlaceholderLostError extends Error { - static [Symbol.hasInstance](value: unknown): boolean { - if (this !== DialCacheRedisPlaceholderLostError) { - return Function.prototype[Symbol.hasInstance].call(this, value); - } - return typeof value === "object" - && value !== null - && Object.getOwnPropertyDescriptor(value, redisPlaceholderLostErrorBrand)?.value === true; - } - - constructor(message: string) { - super(message); - this.name = "DialCacheRedisPlaceholderLostError"; - // CJS adapter subpaths are separate bundles; a global symbol preserves root-export instanceof checks. - Object.defineProperty(this, redisPlaceholderLostErrorBrand, { value: true }); - } -} - /** Serialized cache data, independent of any Redis client or wire framing. */ export type RedisCachePayload = string | Buffer; @@ -91,12 +65,13 @@ export type RedisCachePayload = string | Buffer; * A served Redis frame: the payload bytes past the frame header plus the * header's creation time. The payload is the serializer output, possibly * still wrapped in a compression envelope that DialCache core interprets - * above the adapter (see the `dialcache/redis-protocol` module doc). Tracked - * frames carry Redis server time written by the stamp script; untracked - * frames carry the writer's client clock. DialCache consumes `createdAtMs` - * only for observability (the shadow value-age observation) — tracked - * watermark fencing already happened inside the decoder — so it never - * affects serving decisions. + * above the adapter (see the `dialcache/redis-protocol` module doc). All + * frames carry application-clock time supplied by the writer. Tracked serving + * and initial-shadow reads reject frames dated after the reading process's + * clock before deserialization; confirmation reads retain them for payload + * comparison, and untracked reads treat the stamp as informational. DialCache + * also uses `createdAtMs` for shadow value-age observability. Tracked watermark + * fencing already happened inside the decoder. */ export interface DecodedRedisFrame { readonly payload: RedisCachePayload; @@ -108,15 +83,15 @@ interface RedisValueRequest { readonly valueKey: string; } -interface TrackedRedisValueRequest extends RedisValueRequest { +interface TrackedRedisReadRequest extends RedisValueRequest { readonly watermarkKey: string; } -interface UntrackedRedisValueRequest extends RedisValueRequest { +interface UntrackedRedisReadRequest extends RedisValueRequest { readonly watermarkKey?: never; } -export type RedisReadRequest = TrackedRedisValueRequest | UntrackedRedisValueRequest; +export type RedisReadRequest = TrackedRedisReadRequest | UntrackedRedisReadRequest; /** * Per-use-case read policy supplied by DialCache. Adapters may use the signal @@ -127,17 +102,12 @@ export interface RedisReadContext { readonly signal: AbortSignal; } -interface RedisWriteBase extends RedisValueRequest { +export interface RedisWriteRequest extends RedisValueRequest { /** Positive integer no greater than 31,536,000,000 (365 days). */ readonly cacheTtlMs: number; readonly value: RedisCachePayload; } -type TrackedRedisWriteRequest = RedisWriteBase & TrackedRedisValueRequest; -type UntrackedRedisWriteRequest = RedisWriteBase & UntrackedRedisValueRequest; - -export type RedisWriteRequest = TrackedRedisWriteRequest | UntrackedRedisWriteRequest; - export interface RedisInvalidationRequest { readonly watermarkKey: string; /** Nonnegative integer no greater than 31,536,000,000 (365 days). */ @@ -157,7 +127,8 @@ export interface RedisInvalidationRequest { * * Tracked invalidation also requires the Redis deployment to preserve * watermark keys for their derived TTL. Losing a watermark through eviction, - * failover, restore, or external deletion removes its prior publication fence. + * failover, restore, or external deletion removes its prior read-time + * invalidation fence. */ export interface DialCacheRedisClient { /** @@ -168,11 +139,12 @@ export interface DialCacheRedisClient { * * Raw values are Redis bulk strings (`Buffer`) or null. A missing value, a * frame shorter than the version/timestamp/encoding header, or an - * unsupported frame version is a cache miss. A tracked read also misses - * when its watermark is missing, is not a finite unsigned decimal, or is - * greater than or equal to the frame's creation time. In other words, - * `createdAt <= watermark` is fenced. Unsupported payload encodings and - * non-bulk runtime replies are payload protocol errors rather than misses. + * unsupported frame version is a cache miss. A missing tracked watermark + * is the zero baseline. A tracked read misses when a present watermark is + * not a nonnegative safe-integer decimal or is greater than or equal to the + * frame's creation time. In other words, `createdAt <= watermark` is fenced. + * Unsupported payload encodings and non-bulk runtime replies are payload + * protocol errors rather than misses. * * Tracked implementations must read the value and watermark atomically from * one authoritative snapshot; replica lag must not hide an invalidation. @@ -188,45 +160,29 @@ export interface DialCacheRedisClient { * Write a DialCache Redis frame using the `dialcache/redis-protocol` * encoders, or preserve their exact behavior. * - * Untracked writes are one native `SET valueKey frame PX cacheTtlMs` whose + * All writes are one native `SET valueKey frame PX cacheTtlMs` whose * frame comes from `encodeRedisFrame` with a client-clock `createdAtMs`. - * Untracked reads never consult that stamp for serving or miss decisions, - * but they do surface it as the decoded frame's `createdAtMs`, where it - * feeds the shadow value-age observation — so untracked writers must stamp - * real client time, not a constant. - * - * Tracked writes issue two commands ordered on one connection without a - * transaction: a native `SET` of an `encodeTrackedRedisPlaceholder` frame, - * followed by `WRITE_TRACKED_STAMP_SCRIPT` with `KEYS = [valueKey, - * watermarkKey]` and `ARGV = [cacheTtlMs, nonce]`. Run `cacheTtlMs` through - * `ceilSupportedCacheTtlMs` (exported by `dialcache/redis-protocol`) and - * pass the result as both the SET's `PX` and `ARGV[1]` — `PX` rejects - * fractions and the watermark's lifetime is derived from `ARGV[1]` — and - * the nonce must be the placeholder's. The script fences against the watermark and - * unlinks the value (reply 0), promotes exactly the placeholder carrying - * its nonce to a served frame with server-time `createdAt` (reply 1), or - * reports the placeholder gone (reply 2); it maintains the watermark's - * existence and TTL in the non-fenced cases. Placeholders are unreadable on - * both read paths, so an interleaved or lost stamp degrades to a miss - * bounded by the value TTL — including briefly blanking a previously - * readable key the write replaces — while a delayed stamp of its own - * placeholder remains subject to the invalidation future buffer, like any - * in-flight write. + * DialCache uses a tracked frame's decoded `createdAtMs` for future-time + * rejection and uses decoded timestamps for shadow value-age observations, + * so writers must stamp real client time, not a constant. Untracked serving + * remains governed by Redis TTL and does not reject on the informational + * timestamp. * - * Implementations must not reorder the pair, must mint one placeholder per - * logical write so client-level retries stay paired with their stamp, and - * must surface a SET failure as the write error even when the stamp settled - * (in that case the stamp may have promoted the landed SET, leaving the - * value readable despite the reported failure). Reply 2 must fail the write - * with `DialCacheRedisPlaceholderLostError` so split pairs stay observable; - * after reply 2 the key holds another writer's frame or an unreadable - * placeholder, never this write's value. False means invalidation blocked - * the write. + * Tracked and untracked writes use the same complete-frame SET. Core caps a + * tracked value's physical TTL at one hour. Under the documented clock-skew + * and in-flight-work bounds, invalidation markers outlive every value they + * fence, so writers never read, create, or extend watermarks. */ - write(request: RedisWriteRequest): Awaitable; + write(request: RedisWriteRequest): Awaitable; /** * Advance the watermark monotonically after the source mutation commits. - * Its TTL is derived from the future buffer and any longer existing TTL. + * The adapter supplies a nonnegative safe-integer `Date.now()` sample to + * `INVALIDATE_CACHE_SCRIPT` as `ARGV[2]`; `futureBufferMs` is `ARGV[1]`. + * Reuse that sample through retries within one adapter invocation. + * Its TTL is at least two hours and otherwise derived to outlive the future + * buffer plus the maximum tracked-value TTL. Longer or persistent existing + * markers are preserved. A wrong-type watermark is repaired; any other Redis + * read error surfaces without replacing prior state. */ invalidate(request: RedisInvalidationRequest): Awaitable; } diff --git a/src/redis-protocol.ts b/src/redis-protocol.ts index 10642c0..d5ce8f9 100644 --- a/src/redis-protocol.ts +++ b/src/redis-protocol.ts @@ -1,13 +1,12 @@ /** * Public frame protocol surface for adapter authors and out-of-band tooling. * - * These exports encode frames and mint tracked placeholders (use - * `encodeRedisFrame` and `encodeTrackedRedisPlaceholder` rather than - * reimplementing them — see the latter's JSDoc for the nonce contract), - * decode a frame into its payload bytes and header creation time, resolve - * and validate mutation replies, guard the write-TTL acceptance domain, and - * carry the tracked stamp and invalidation Lua sources the bundled adapters - * dispatch. The + * These exports encode and decode complete frames, validate mutation replies, + * guard the write-TTL acceptance domain, and carry the invalidation Lua source + * the bundled adapters dispatch. The invalidation script accepts + * `[futureBufferMs, invalidatedAtMs]`; both are nonnegative safe integers, and + * the application-clock timestamp must remain stable through one logical + * dispatch and its recovery. The * payload region past the header is opaque at this layer: entries written by * DialCache releases with payload compression may begin with a compression * envelope byte (0x00 escape, 0x01/0x02 zstd; see the README Compression @@ -15,20 +14,14 @@ * never decompress or otherwise rewrite payload bytes. */ export { ceilSupportedCacheTtlMs } from "./internal/duration.js"; -export { - INVALIDATE_CACHE_SCRIPT, - WRITE_TRACKED_STAMP_SCRIPT, -} from "./internal/redis-scripts.js"; +export { INVALIDATE_CACHE_SCRIPT } from "./internal/redis-scripts.js"; export { decodeRedisFrame, decodeTrackedRedisFrame, encodeRedisFrame, - encodeTrackedRedisPlaceholder, - type TrackedRedisPlaceholder, } from "./internal/redis-payload.js"; export type { DecodedRedisFrame } from "./redis-client.js"; export { - resolveTrackedRedisWriteReply, validateRedisScriptInvalidationReply, validateRedisSetReply, } from "./internal/redis-script-reply.js"; diff --git a/src/valkey-glide.ts b/src/valkey-glide.ts index 7b33915..1ad2c92 100644 --- a/src/valkey-glide.ts +++ b/src/valkey-glide.ts @@ -1,18 +1,16 @@ -import { createHash } from "node:crypto"; - import { ceilSupportedCacheTtlMs } from "./internal/duration.js"; import { + buildRedisInvalidationScriptArguments, + INVALIDATE_CACHE_SCRIPT, + INVALIDATE_CACHE_SCRIPT_SHA1, +} from "./internal/redis-invalidation.js"; +import { + assertValidRedisTimestampMs, decodeRedisFrame, decodeTrackedRedisFrame, encodeRedisFrame, - encodeTrackedRedisPlaceholder, } from "./internal/redis-payload.js"; import { - INVALIDATE_CACHE_SCRIPT, - WRITE_TRACKED_STAMP_SCRIPT, -} from "./internal/redis-scripts.js"; -import { - resolveTrackedRedisWriteReply, validateRedisScriptInvalidationReply, validateRedisSetReply, } from "./internal/redis-script-reply.js"; @@ -20,20 +18,7 @@ import { DialCacheRedisPayloadError, type DialCacheRedisClient } from "./redis-c type ValkeyGlideString = string | Buffer; -// Redis caches EVAL'd sources under sha1(source), so these digests are by -// definition the ones the EVALSHA dispatches must use and the ones the EVAL -// recoveries repopulate. -const WRITE_TRACKED_STAMP_SHA1 = createHash("sha1").update(WRITE_TRACKED_STAMP_SCRIPT).digest("hex"); -const INVALIDATE_CACHE_SHA1 = createHash("sha1").update(INVALIDATE_CACHE_SCRIPT).digest("hex"); - -// Matches the server's raw NOSCRIPT reply and GLIDE's mapped NoScriptError -// wording, case-insensitively so message-format drift cannot blind it. -function isNoScriptError(error: Error): boolean { - return error.message.toLowerCase().includes("noscript"); -} - interface ValkeyGlideBatch { - customCommand(args: ValkeyGlideString[]): ValkeyGlideBatch; mget(keys: ValkeyGlideString[]): ValkeyGlideBatch; } @@ -66,8 +51,6 @@ interface ValkeyGlideClientIdentity { export interface ValkeyGlideRuntime { /** The Batch constructor exported by the same GLIDE module instance as the client. */ readonly Batch: new (isAtomic: boolean) => ValkeyGlideBatch; - /** The ClusterBatch constructor exported by the same GLIDE module instance as the client. */ - readonly ClusterBatch: new (isAtomic: boolean) => ValkeyGlideBatch; /** The standalone client class exported by the same GLIDE module instance as the client. */ readonly GlideClient: ValkeyGlideClientIdentity; /** The cluster client class exported by the same GLIDE module instance as the client. */ @@ -129,16 +112,11 @@ function classifyValkeyGlideClient( * read deadline may return before this adapter's invocation settles. Tracked * standalone reads use a one-command primary batch, while tracked cluster * reads route MGET explicitly to the slot primary, so replica lag cannot hide - * an invalidation watermark. Both mutation scripts dispatch as EVALSHA by - * their source SHA1 and recover a flushed script cache by re-sending the - * source as EVAL — which the server caches under that same SHA1 — so the - * first mutation against a cold script cache pays one extra round trip. - * Tracked writes batch a native placeholder SET with the stamp EVALSHA; - * cluster write batches route to the slot primary. Batches are deliberately - * non-atomic: MGET and SET are atomic themselves, an interleaved stamp is - * safe by design, and MULTI/EXEC would consume caller-owned WATCH state. - * Recovery differs by script: the stamp is retried only on NOSCRIPT, while - * invalidation retries any rejection once with EVAL by source. When that + * an invalidation watermark. Every write is one native SET of a complete + * client-stamped frame, routed to the slot primary for cluster clients. + * Invalidation dispatches as EVALSHA by its source SHA1 and retries any + * rejection once by re-sending the source as EVAL, which also repopulates a + * flushed script cache. When that * retry also fails, the original rejection is attached as the retry error's * `cause` unless it already carries one. */ @@ -146,9 +124,9 @@ export function createValkeyGlideDialCacheClient( client: ValkeyGlideScriptingClient, glide: ValkeyGlideRuntime, ): DialCacheRedisClient { - if (typeof glide.Batch !== "function" || typeof glide.ClusterBatch !== "function") { + if (typeof glide.Batch !== "function") { throw new Error( - "Valkey GLIDE DialCache requires @valkey/valkey-glide >=2.0.0 with Batch and ClusterBatch constructors", + "Valkey GLIDE DialCache requires @valkey/valkey-glide >=2.0.0 with a Batch constructor", ); } const isCluster = classifyValkeyGlideClient(client, glide) === "cluster"; @@ -188,65 +166,26 @@ export function createValkeyGlideDialCacheClient( return decodeTrackedRedisFrame(pair[0], pair[1]); }, async write(request) { - const { valueKey, watermarkKey, value } = request; + const { valueKey, value } = request; const cacheTtlMs = ceilSupportedCacheTtlMs(request.cacheTtlMs); const execOptions = keyedOptions(valueKey); - - if (watermarkKey === undefined) { - const frame = encodeRedisFrame(value, Date.now()); - validateRedisSetReply( - await client.customCommand(["SET", valueKey, frame, "PX", String(cacheTtlMs)], execOptions), - ); - return true; - } - - const { frame, nonce } = encodeTrackedRedisPlaceholder(value); - const stampArgs: ValkeyGlideString[] = [String(cacheTtlMs), nonce]; - const batch = (isCluster ? new glide.ClusterBatch(false) : new glide.Batch(false)) - .customCommand(["SET", valueKey, frame, "PX", String(cacheTtlMs)]) - .customCommand([ - "EVALSHA", - WRITE_TRACKED_STAMP_SHA1, - "2", - valueKey, - watermarkKey, - ...stampArgs, - ]); - const replies = await client.exec(batch, false, execOptions); - if (!Array.isArray(replies) || replies.length !== 2) { - throw new DialCacheRedisPayloadError("Invalid DialCache Redis write reply"); - } - const [setReply, rawStamp] = replies as [unknown, unknown]; - // A failed SET is the write outcome even when the stamp settled. - if (setReply instanceof Error) { - throw setReply; - } - validateRedisSetReply(setReply); - let stampReply: unknown = rawStamp; - if (rawStamp instanceof Error) { - if (!isNoScriptError(rawStamp)) { - throw rawStamp; - } - // Only NOSCRIPT proves the batched stamp never executed, so only it - // is retried: after any other error a re-run could find its own - // frame already promoted and misreport the write as a lost - // placeholder. EVAL resends the source, the server caches it under - // the same SHA1 the batched EVALSHA uses, and the nonce keeps the - // late stamp paired to this write. - stampReply = await client.customCommand( - ["EVAL", WRITE_TRACKED_STAMP_SCRIPT, "2", valueKey, watermarkKey, ...stampArgs], - execOptions, - ); - } - return resolveTrackedRedisWriteReply(stampReply); + const frame = encodeRedisFrame(value, Date.now()); + validateRedisSetReply( + await client.customCommand(["SET", valueKey, frame, "PX", String(cacheTtlMs)], execOptions), + ); }, async invalidate({ watermarkKey, futureBufferMs }) { - const invalidateArgs: ValkeyGlideString[] = [String(futureBufferMs)]; + const invalidatedAtMs = Date.now(); + assertValidRedisTimestampMs(invalidatedAtMs); + const invalidateArgs = buildRedisInvalidationScriptArguments( + futureBufferMs, + invalidatedAtMs, + ); const options = keyedOptions(watermarkKey); let raw: unknown; try { raw = await client.customCommand( - ["EVALSHA", INVALIDATE_CACHE_SHA1, "1", watermarkKey, ...invalidateArgs], + ["EVALSHA", INVALIDATE_CACHE_SCRIPT_SHA1, "1", watermarkKey, ...invalidateArgs], options, ); } catch (error) { diff --git a/test/datadog.test.ts b/test/datadog.test.ts index fbb6b93..5f7a538 100644 --- a/test/datadog.test.ts +++ b/test/datadog.test.ts @@ -96,6 +96,7 @@ const ERROR_KINDS: Readonly> = { cache_read: true, cache_read_timeout: true, cache_write: true, + tracked_ttl_clamped: true, serialization_load: true, serialization_dump: true, compression: true, @@ -109,7 +110,6 @@ const SHADOW_VALIDATION_OUTCOMES: Readonly mismatch: true, superseded: true, filled: true, - fill_blocked: true, fill_error: true, redis_error: true, source_error: true, @@ -169,6 +169,7 @@ describe("Datadog metrics adapter", () => { }, 42.5, ); + metrics.observeFutureTimestampOffset(cacheLabels, 0.007); metrics.compression({ ...cacheLabels, outcome: "compressed" }); metrics.observeGet(cacheLabels, 0.125); metrics.observeFallback(cacheLabels, 0.5); @@ -218,6 +219,12 @@ describe("Datadog metrics adapter", () => { value: 42.5, tags: { cache_namespace: "users", use_case: "LoadUser", key_type: "user_id", outcome: "mismatch" }, }, + { + method: "distribution", + name: "dialcache.future_timestamp_offset", + value: 0.007, + tags: baseTags, + }, { method: "increment", name: "dialcache.compression.count", @@ -271,6 +278,7 @@ describe("Datadog metrics adapter", () => { }, 60, ); + metrics.observeFutureTimestampOffset(cacheLabels, 0.006); expect(client.calls.map(({ method, name, value }) => ({ method, name, value }))).toEqual([ { method: observationMetricType, name: "service.cache.get.duration", value: 0.01 }, @@ -281,6 +289,7 @@ describe("Datadog metrics adapter", () => { { method: observationMetricType, name: "service.cache.compression.ratio", value: 0.04 }, { method: observationMetricType, name: "service.cache.compression.duration", value: 0.05 }, { method: observationMetricType, name: "service.cache.shadow.value_age", value: 60 }, + { method: observationMetricType, name: "service.cache.future_timestamp_offset", value: 0.006 }, ]); }); } @@ -461,7 +470,7 @@ describe("Datadog metrics adapter", () => { error.name = rawErrorName; throw error; }, - write: async () => true, + write: async () => {}, invalidate: async () => undefined, }; const dialcache = new DialCache({ @@ -575,15 +584,15 @@ describe("Datadog metrics adapter", () => { it("enforces Datadog's 200-character final metric-name limit", () => { const client = new RecordingDogStatsDClient(); - const longestValidNamespace = "a".repeat(177); - const tooLongNamespace = "a".repeat(178); + const longestValidNamespace = "a".repeat(176); + const tooLongNamespace = "a".repeat(177); const metrics = new DatadogDialCacheMetrics({ client, namespace: longestValidNamespace, observationMetricType: "distribution", }); - metrics.observeSerialization({ ...cacheLabels, operation: "dump" }, 1); + metrics.observeFutureTimestampOffset(cacheLabels, 1); expect(client.calls[0]?.name).toHaveLength(200); expect( diff --git a/test/dialcache-invalidation.test.ts b/test/dialcache-invalidation.test.ts index 9bdb024..3523c8c 100644 --- a/test/dialcache-invalidation.test.ts +++ b/test/dialcache-invalidation.test.ts @@ -15,6 +15,11 @@ import { type SerializationMetricLabels, type Serializer, } from "../src/index.js"; +import { + MAX_SUPPORTED_DURATION_MS, + MAX_TRACKED_REDIS_VALUE_TTL_MS, +} from "../src/internal/duration.js"; +import { MIN_WATERMARK_TTL_MS } from "../src/internal/redis-scripts.js"; import { encodeFrame, FakeRedis } from "./fake-redis.js"; class RecordingMetrics implements DialCacheMetricsAdapter { @@ -71,7 +76,6 @@ const localAndRemote = (ttlSec = 60) => DialCacheKeyConfig.enabled(ttlSec); const valueKey = (useCase: string, args = ""): string => `{urn:user_id:123}${args}#${useCase}:dialcache-frame-v1`; const watermarkKey = "{urn:user_id:123}#watermark"; const MAX_CACHE_TTL_SEC = 31_536_000; -const MAX_SUPPORTED_DURATION_MS = 31_536_000_000; const WATERMARK_TTL_MARGIN_MS = 60_000; describe("DialCache targeted invalidation watermarks", () => { @@ -118,7 +122,7 @@ describe("DialCache targeted invalidation watermarks", () => { expect(redis.readWatermarkValue(watermarkKey)).toBe(Date.parse("2026-05-12T18:00:00.000Z")); }); - it("does not write remote or local cache during a future invalidation window", async () => { + it("stores a complete remote frame but does not publish local cache during a future invalidation window", async () => { const redis = new FakeRedis(); const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); let calls = 0; @@ -137,10 +141,20 @@ describe("DialCache targeted invalidation watermarks", () => { expect(first).toEqual({ userId: "123", calls: 1 }); expect(second).toEqual({ userId: "123", calls: 2 }); - expect([...redis.values.keys()]).toEqual([watermarkKey]); + expect([...redis.values.keys()].sort()).toEqual([ + watermarkKey, + valueKey("FutureBufferUser"), + ].sort()); + await expect(redis.read({ + valueKey: valueKey("FutureBufferUser"), + watermarkKey, + })).resolves.toBeNull(); + await expect(redis.read({ valueKey: valueKey("FutureBufferUser") })).resolves.toMatchObject({ + payload: JSON.stringify(second), + }); }); - it("rejects a write when invalidation arrives during fallback", async () => { + it("stores but fences a write when invalidation arrives during fallback", async () => { const redis = new FakeRedis(); const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); let calls = 0; @@ -164,10 +178,17 @@ describe("DialCache targeted invalidation watermarks", () => { expect(first).toEqual({ userId: "123", calls: 1 }); expect(second).toEqual({ userId: "123", calls: 2 }); - expect([...redis.values.keys()]).toEqual([watermarkKey]); + expect([...redis.values.keys()].sort()).toEqual([ + watermarkKey, + valueKey("FutureBufferFallbackRace"), + ].sort()); + await expect(redis.read({ + valueKey: valueKey("FutureBufferFallbackRace"), + watermarkKey, + })).resolves.toBeNull(); }); - it("rejects a write when invalidation remains active after slow serialization", async () => { + it("stores but fences a write when invalidation remains active after slow serialization", async () => { const redis = new FakeRedis(); let signalDumpStarted = (): void => undefined; const dumpStarted = new Promise((resolve) => { @@ -209,10 +230,17 @@ describe("DialCache targeted invalidation watermarks", () => { expect(first).toEqual({ userId: "123", calls: 1 }); expect(second).toEqual({ userId: "123", calls: 2 }); - expect([...redis.values.keys()]).toEqual([watermarkKey]); + expect([...redis.values.keys()].sort()).toEqual([ + watermarkKey, + valueKey("FutureBufferSerializationRace"), + ].sort()); + await expect(redis.read({ + valueKey: valueKey("FutureBufferSerializationRace"), + watermarkKey, + })).resolves.toBeNull(); }); - it("blocks a same-millisecond write for a zero-length future buffer", async () => { + it("fences a same-millisecond complete write for a zero-length future buffer", async () => { const redis = new FakeRedis(); const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); let calls = 0; @@ -225,18 +253,18 @@ describe("DialCache targeted invalidation watermarks", () => { }); await dialcache.invalidateRemote("user_id", "123", 0); - const blocked = await dialcache.enable(async () => await getUser("123")); + const fenced = await dialcache.enable(async () => await getUser("123")); vi.advanceTimersByTime(1); const written = await dialcache.enable(async () => await getUser("123")); const cached = await dialcache.enable(async () => await getUser("123")); - expect(blocked).toEqual({ userId: "123", calls: 1 }); + expect(fenced).toEqual({ userId: "123", calls: 1 }); expect(written).toEqual({ userId: "123", calls: 2 }); expect(cached).toEqual(written); expect(calls).toBe(2); }); - it("resumes tracked writes after the future buffer", async () => { + it("serves tracked writes after the future buffer", async () => { const redis = new FakeRedis(); const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); let calls = 0; @@ -258,7 +286,7 @@ describe("DialCache targeted invalidation watermarks", () => { expect(redis.values.has(valueKey("FutureBufferExpires"))).toBe(true); }); - it("treats a tracked value with a missing watermark marker as a miss", async () => { + it("treats a missing tracked watermark as the zero baseline", async () => { const redis = new FakeRedis(); redis.setRaw(valueKey("MissingWatermark"), encodeFrame({ source: "stale" })); const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); @@ -274,9 +302,10 @@ describe("DialCache targeted invalidation watermarks", () => { const first = await dialcache.enable(async () => await getUser("123")); const second = await dialcache.enable(async () => await getUser("123")); - expect(first).toEqual({ userId: "123", source: "fallback-1" }); - expect(second).toEqual({ userId: "123", source: "fallback-1" }); - expect(redis.readWatermarkValue(watermarkKey)).toBe(0); + expect(first).toEqual({ source: "stale" }); + expect(second).toEqual(first); + expect(calls).toBe(0); + expect(redis.readWatermarkValue(watermarkKey)).toBeNull(); }); it("preserves the furthest watermark across repeated invalidations", async () => { @@ -291,7 +320,7 @@ describe("DialCache targeted invalidation watermarks", () => { expect(redis.readWatermarkValue(watermarkKey)).toBe(first); }); - it("derives watermark lifetime from value TTLs and invalidations without extending it on reads", async () => { + it("creates watermarks only on invalidation and does not extend them on reads or writes", async () => { const redis = new FakeRedis(); const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); const getUser = dialcache.cached(async (userId: string) => ({ userId }), { @@ -303,19 +332,20 @@ describe("DialCache targeted invalidation watermarks", () => { }); await dialcache.enable(async () => await getUser("123")); - const afterWrite = redis.ttlMs(watermarkKey); - vi.advanceTimersByTime(1_000); - await dialcache.enable(async () => await getUser("123")); - const afterRead = redis.ttlMs(watermarkKey); + expect(redis.ttlMs(watermarkKey)).toBe(-2); + expect(redis.ttlMs(valueKey("WatermarkLifetime"))).toBe(MAX_TRACKED_REDIS_VALUE_TTL_MS); + await dialcache.invalidateRemote("user_id", "123"); const afterInvalidation = redis.ttlMs(watermarkKey); + vi.advanceTimersByTime(1_000); + await dialcache.enable(async () => await getUser("123")); + const afterReadAndWrite = redis.ttlMs(watermarkKey); - expect(afterWrite).toBe(2 * 60 * 60 * 1_000 + 60_000); - expect(afterRead).toBe(afterWrite - 1_000); - expect(afterInvalidation).toBe(afterRead); + expect(afterInvalidation).toBe(MIN_WATERMARK_TTL_MS); + expect(afterReadAndWrite).toBe(afterInvalidation - 1_000); }); - it("keeps one shared watermark alive for the longest outstanding tracked value", async () => { + it("does not create or extend a shared watermark for tracked values with different TTLs", async () => { const redis = new FakeRedis(); const dialcache = new DialCache({ redis: { client: redis } }); const getLongLived = dialcache.cached(async (userId: string) => ({ userId, lifetime: "long" }), { @@ -334,12 +364,16 @@ describe("DialCache targeted invalidation watermarks", () => { }); await dialcache.enable(async () => await getLongLived("123")); - const afterLongWrite = redis.ttlMs(watermarkKey); + expect(redis.ttlMs(watermarkKey)).toBe(-2); + expect(redis.ttlMs(valueKey("LongWatermarkLifetime"))).toBe(MAX_TRACKED_REDIS_VALUE_TTL_MS); + + await dialcache.invalidateRemote("user_id", "123"); + const afterInvalidation = redis.ttlMs(watermarkKey); vi.advanceTimersByTime(60 * 60 * 1_000); await dialcache.enable(async () => await getShortLived("123")); - expect(afterLongWrite).toBe(2 * 60 * 60 * 1_000 + 60_000); - expect(redis.ttlMs(watermarkKey)).toBe(afterLongWrite - 60 * 60 * 1_000); + expect(afterInvalidation).toBe(MIN_WATERMARK_TTL_MS); + expect(redis.ttlMs(watermarkKey)).toBe(afterInvalidation - 60 * 60 * 1_000); }); it("fails open without caching when tracked watermark reads fail", async () => { @@ -494,9 +528,10 @@ describe("DialCache targeted invalidation watermarks", () => { expect(metrics.events).toEqual([]); }); - it("accepts the maximum TTL across local, Redis, and tracked-watermark storage", async () => { + it("caps tracked Redis values at one hour while retaining the configured local TTL", async () => { const redis = new FakeRedis(); - const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); + const metrics = new RecordingMetrics(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); let calls = 0; const getUser = dialcache.cached(async (id: string) => ({ id, calls: ++calls }), { keyType: "user_id", @@ -508,14 +543,58 @@ describe("DialCache targeted invalidation watermarks", () => { const first = await dialcache.enable(async () => await getUser("123")); const second = await dialcache.enable(async () => await getUser("123")); + expect(redis.ttlMs(valueKey("MaximumSupportedTtl"))).toBe(MAX_TRACKED_REDIS_VALUE_TTL_MS); + expect(redis.ttlMs(watermarkKey)).toBe(-2); + expect(metrics.events.filter(({ name }) => name === "error").map(({ labels }) => labels)).toEqual([ + { + cacheNamespace: "urn", + useCase: "MaximumSupportedTtl", + keyType: "user_id", + layer: CacheLayer.REMOTE, + error: "tracked_ttl_clamped", + inFallback: false, + }, + ]); + + vi.advanceTimersByTime(MAX_TRACKED_REDIS_VALUE_TTL_MS + 1); + const third = await dialcache.enable(async () => await getUser("123")); - expect(second).toBe(first); + expect(second).toEqual(first); + expect(second).not.toBe(first); + expect(third).toBe(second); expect(calls).toBe(1); - expect(redis.mGetCalls).toBe(1); - expect(redis.ttlMs(valueKey("MaximumSupportedTtl"))).toBe(MAX_SUPPORTED_DURATION_MS); - expect(redis.ttlMs(watermarkKey)).toBe( - MAX_SUPPORTED_DURATION_MS + WATERMARK_TTL_MARGIN_MS, - ); + expect(redis.mGetCalls).toBe(2); + }); + + it("caps only tracked Redis TTLs above one hour", async () => { + const redis = new FakeRedis(); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); + const trackedTtlSec = MAX_TRACKED_REDIS_VALUE_TTL_MS / 1_000 - 1; + const untrackedTtlSec = 2 * MAX_TRACKED_REDIS_VALUE_TTL_MS / 1_000; + const getTracked = dialcache.cached(async (id: string) => ({ id }), { + keyType: "user_id", + useCase: "TrackedBelowCap", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: remoteOnly(trackedTtlSec), + }); + const getUntracked = dialcache.cached(async (id: string) => ({ id }), { + keyType: "user_id", + useCase: "UntrackedAboveCap", + cacheKey: (id) => id, + defaultConfig: remoteOnly(untrackedTtlSec), + }); + + await dialcache.enable(async () => await getTracked("123")); + await dialcache.enable(async () => await getUntracked("123")); + + const untrackedValueKey = `${new DialCacheKey({ + keyType: "user_id", + id: "123", + useCase: "UntrackedAboveCap", + }).urn}:dialcache-frame-v1`; + expect(redis.ttlMs(valueKey("TrackedBelowCap"))).toBe(trackedTtlSec * 1_000); + expect(redis.ttlMs(untrackedValueKey)).toBe(untrackedTtlSec * 1_000); }); it("accepts the maximum future buffer and derives its watermark TTL safely", async () => { @@ -529,7 +608,7 @@ describe("DialCache targeted invalidation watermarks", () => { Date.now() + MAX_SUPPORTED_DURATION_MS, ); expect(redis.ttlMs(watermarkKey)).toBe( - MAX_SUPPORTED_DURATION_MS + WATERMARK_TTL_MARGIN_MS, + MAX_SUPPORTED_DURATION_MS + MAX_TRACKED_REDIS_VALUE_TTL_MS + WATERMARK_TTL_MARGIN_MS, ); }); @@ -560,7 +639,7 @@ describe("DialCache targeted invalidation watermarks", () => { ); }); - it("documents that Redis invalidation does not evict local cache", async () => { + it("documents that Redis invalidation does not evict a validated local cache entry", async () => { const redis = new FakeRedis(); const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 } }); let version = 1; @@ -573,11 +652,13 @@ describe("DialCache targeted invalidation watermarks", () => { }); const before = await dialcache.enable(async () => await getUser("123")); + const warmed = await dialcache.enable(async () => await getUser("123")); version = 2; await dialcache.invalidateRemote("user_id", "123"); const after = await dialcache.enable(async () => await getUser("123")); expect(before).toEqual({ userId: "123", version: 1 }); + expect(warmed).toEqual(before); expect(after).toEqual({ userId: "123", version: 1 }); }); diff --git a/test/dialcache-liveness.test.ts b/test/dialcache-liveness.test.ts index 8f31305..e48a416 100644 --- a/test/dialcache-liveness.test.ts +++ b/test/dialcache-liveness.test.ts @@ -547,7 +547,7 @@ describe("DialCache fallback liveness", () => { readStarted.resolve(); return await readGate.promise; }, - write: async () => true, + write: async () => {}, invalidate: async () => undefined, }; const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 200 } }); @@ -589,7 +589,7 @@ describe("DialCache fallback liveness", () => { }; const redis: DialCacheRedisClient = { read: async () => ({ payload: "stored", createdAtMs: Date.now() }), - write: async () => true, + write: async () => {}, invalidate: async () => undefined, }; const dialcache = new DialCache({ redis: { client: redis }, metrics }); @@ -621,7 +621,7 @@ describe("DialCache fallback liveness", () => { it("does not apply a completed fallback's deadline to serializer dump or Redis write", async () => { const dumpGate = deferred(); const dumpStarted = deferred(); - const writeGate = deferred(); + const writeGate = deferred(); const writeStarted = deferred(); const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); const serializer: Serializer = { @@ -673,7 +673,7 @@ describe("DialCache fallback liveness", () => { expect(settled).toBe(false); expect(dialcache.getCoalescingState().process.activeLeaders).toBe(1); - writeGate.resolve(true); + writeGate.resolve(); await expect(result).resolves.toBe("value"); expect(dialcache.getCoalescingState().process.activeLeaders).toBe(0); }); diff --git a/test/dialcache-logger.test.ts b/test/dialcache-logger.test.ts index a7df3e1..ebb3509 100644 --- a/test/dialcache-logger.test.ts +++ b/test/dialcache-logger.test.ts @@ -160,7 +160,7 @@ describe("DialCache logger isolation", () => { const invalidationError = new Error("invalidation failed"); const redis = { read: vi.fn(async () => null), - write: vi.fn(async () => true), + write: vi.fn(async () => {}), invalidate: vi.fn(async () => { throw invalidationError; }), diff --git a/test/dialcache-metrics.test.ts b/test/dialcache-metrics.test.ts index 15f5fa1..23879dc 100644 --- a/test/dialcache-metrics.test.ts +++ b/test/dialcache-metrics.test.ts @@ -96,6 +96,7 @@ describe("DialCache observability metrics", () => { invalidation: vi.fn(() => thenable), coalesced: vi.fn(() => thenable), shadowValidation: vi.fn(() => thenable), + observeFutureTimestampOffset: vi.fn(() => thenable), observeGet: vi.fn(() => thenable), observeFallback: vi.fn(() => thenable), observeSerialization: vi.fn(() => thenable), @@ -133,6 +134,7 @@ describe("DialCache observability metrics", () => { keyType: "user_id", outcome: "match", } satisfies ShadowValidationMetricLabels); + isolatedMetrics.observeFutureTimestampOffset?.(labels, 0.001); isolatedMetrics.observeGet(labels, 0); isolatedMetrics.observeFallback(labels, 0); isolatedMetrics.observeSerialization({ ...labels, operation: "dump" }, 0); @@ -140,7 +142,7 @@ describe("DialCache observability metrics", () => { expect(then).not.toHaveBeenCalled(); await tick(); - expect(then).toHaveBeenCalledTimes(11); + expect(then).toHaveBeenCalledTimes(12); }); it("includes the configured cache namespace on every metric path", async () => { @@ -420,7 +422,7 @@ describe("DialCache observability metrics", () => { read: vi.fn(async () => { throw cacheError; }), - write: vi.fn(async () => true), + write: vi.fn(async () => {}), invalidate: vi.fn(async () => undefined), }; const cacheFailure = new DialCache({ redis: { client: failingRedis, readTimeoutMs: 1_000 }, metrics, logger }); diff --git a/test/dialcache-redis-read-deadline.test.ts b/test/dialcache-redis-read-deadline.test.ts index 211c56c..b27fb24 100644 --- a/test/dialcache-redis-read-deadline.test.ts +++ b/test/dialcache-redis-read-deadline.test.ts @@ -62,7 +62,7 @@ function redisClient(read: DialCacheRedisClient["read"]): { readonly write: ReturnType>; } { const readMock = vi.fn(read); - const write = vi.fn(async () => true); + const write = vi.fn(async () => {}); return { client: { read: readMock, @@ -180,6 +180,90 @@ describe("DialCache Redis read deadlines", () => { ).not.toThrow(); }); + it.each([ + { + name: "forward clock step", + dispatchNowMs: 1_000, + settledNowMs: 1_200, + frameCreatedAtMs: 1_100, + expectedSource: "redis", + expectedOffsetSeconds: null, + }, + { + name: "backward clock step", + dispatchNowMs: 1_200, + settledNowMs: 1_000, + frameCreatedAtMs: 1_100, + expectedSource: "fallback", + expectedOffsetSeconds: 0.1, + }, + ])("samples the reader clock after a bounded read settles across a $name", async ({ + dispatchNowMs, + settledNowMs, + frameCreatedAtMs, + expectedSource, + expectedOffsetSeconds, + }) => { + vi.mocked(performance.now).mockReturnValue(0); + let readerNowMs = dispatchNowMs; + const nowSpy = vi.spyOn(Date, "now").mockImplementation(() => readerNowMs); + const readStarted = deferred(); + const readGate = deferred(); + const redis = redisClient(async () => { + readStarted.resolve(undefined); + return await readGate.promise; + }); + const observeFutureTimestampOffset = vi.fn< + NonNullable + >(); + const metrics = { + ...metricsWithError(vi.fn()), + observeFutureTimestampOffset, + }; + const dialcache = new DialCache({ + redis: { client: redis.client, readTimeoutMs: 100 }, + metrics, + }); + const fallback = vi.fn(async () => ({ source: "fallback" })); + const load = dialcache.cached(fallback, { + keyType: "id", + useCase: "RedisReadSettledClockSample", + cacheKey: () => "123", + trackForInvalidation: true, + defaultConfig: remoteConfig, + }); + + const result = dialcache.enable(async () => await load()); + await readStarted.promise; + expect(nowSpy).not.toHaveBeenCalled(); + readerNowMs = settledNowMs; + readGate.resolve({ + payload: JSON.stringify({ source: "redis" }), + createdAtMs: frameCreatedAtMs, + }); + + await expect(result).resolves.toEqual({ source: expectedSource }); + + expect(nowSpy).toHaveBeenCalledTimes(1); + expect(fallback).toHaveBeenCalledTimes(expectedSource === "fallback" ? 1 : 0); + if (expectedOffsetSeconds === null) { + expect(observeFutureTimestampOffset).not.toHaveBeenCalled(); + expect(redis.write).not.toHaveBeenCalled(); + } else { + expect(observeFutureTimestampOffset).toHaveBeenCalledOnce(); + expect(observeFutureTimestampOffset).toHaveBeenCalledWith( + { + cacheNamespace: "urn", + useCase: "RedisReadSettledClockSample", + keyType: "id", + layer: CacheLayer.REMOTE, + }, + expectedOffsetSeconds, + ); + expect(redis.write).toHaveBeenCalledOnce(); + } + }); + it("rejects invalid static use-case overrides before reserving the use-case name", () => { const client = redisClient(async () => null).client; const invalidValues: readonly unknown[] = [ @@ -614,10 +698,16 @@ describe("DialCache Redis read deadlines", () => { : { payload: JSON.stringify({ source: "redis" }), createdAtMs: Date.now() }; }); const error = vi.fn(); + const observeFutureTimestampOffset = vi.fn< + NonNullable + >(); const logger = { debug: vi.fn(), warn: vi.fn(), error: vi.fn() }; const dialcache = new DialCache({ redis: { client: redis.client, readTimeoutMs: 10 }, - metrics: metricsWithError(error), + metrics: { + ...metricsWithError(error), + observeFutureTimestampOffset, + }, logger, }); const fallback = vi.fn(async () => ({ source: "fallback" })); @@ -634,7 +724,7 @@ describe("DialCache Redis read deadlines", () => { await expect(dialcache.enable(async () => await load())).resolves.toEqual({ source: "redis" }); if (settlement === "fulfillment") { - firstRead.resolve({ payload: JSON.stringify({ source: "late" }), createdAtMs: Date.now() }); + firstRead.resolve({ payload: JSON.stringify({ source: "late" }), createdAtMs: Date.now() + 1_000 }); } else { firstRead.reject(new Error("late Redis failure")); } @@ -645,6 +735,7 @@ describe("DialCache Redis read deadlines", () => { expect(fallback).toHaveBeenCalledTimes(1); expect(logger.warn).toHaveBeenCalledTimes(1); expect(error).toHaveBeenCalledTimes(1); + expect(observeFutureTimestampOffset).not.toHaveBeenCalled(); }, ); diff --git a/test/dialcache-redis.test.ts b/test/dialcache-redis.test.ts index 3c9abfd..69663e1 100644 --- a/test/dialcache-redis.test.ts +++ b/test/dialcache-redis.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { CacheLayer, @@ -6,6 +6,7 @@ import { DialCacheKey, DialCacheKeyConfig, DialCacheRedisPayloadEncodingError, + type DialCacheMetricsAdapter, type DialCacheRedisClient, type RedisConfig, type Serializer, @@ -17,11 +18,33 @@ const keyFor = (id: string, useCase: string, trackForInvalidation = false): Dial const redisKeyFor = (id: string, useCase: string, trackForInvalidation = false): string => `${keyFor(id, useCase, trackForInvalidation).urn}:dialcache-frame-v1`; +function metricsWithFutureTimestampObserver( + observeFutureTimestampOffset: NonNullable, +): DialCacheMetricsAdapter { + return { + request: vi.fn(), + miss: vi.fn(), + disabled: vi.fn(), + error: vi.fn(), + invalidation: vi.fn(), + observeFutureTimestampOffset, + observeGet: vi.fn(), + observeFallback: vi.fn(), + observeSerialization: vi.fn(), + observeSize: vi.fn(), + }; +} + describe("DialCache Redis TTL layer", () => { beforeEach(() => { vi.useRealTimers(); }); + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + it("reads local miss from Redis and populates the local layer", async () => { // Given one process has already written a value into the shared Redis cache. const redis = new FakeRedis(); @@ -149,6 +172,227 @@ describe("DialCache Redis TTL layer", () => { }); }); + it("rejects a future-dated tracked frame before deserialization and observes one exact offset", async () => { + const nowMs = 1_700_000_000_000; + vi.spyOn(Date, "now").mockReturnValue(nowMs); + const redis = new FakeRedis(); + const useCase = "RedisTrackedFutureFrame"; + const key = keyFor("123", useCase, true); + redis.setRaw( + `${key.urn}:dialcache-frame-v1`, + encodeFrame(JSON.stringify({ source: "redis" }), nowMs + 1_250), + ); + redis.setRaw(`${key.prefix}#watermark`, "0"); + const observeFutureTimestampOffset = vi.fn(); + const metrics = metricsWithFutureTimestampObserver(observeFutureTimestampOffset); + const serializer: Serializer<{ readonly source: string }> = { + dump: vi.fn((value) => JSON.stringify(value)), + load: vi.fn(() => { + throw new Error("future frames must not be deserialized"); + }), + }; + const fallback = vi.fn(async () => ({ source: "fallback" })); + const dialcache = new DialCache({ + redis: { client: redis, readTimeoutMs: 1_000 }, + metrics, + }); + const getUser = dialcache.cached(fallback, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + trackForInvalidation: true, + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + ramp: { [CacheLayer.REMOTE]: 100 }, + }), + serializer, + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual({ source: "fallback" }); + + expect(serializer.load).not.toHaveBeenCalled(); + expect(fallback).toHaveBeenCalledOnce(); + expect(observeFutureTimestampOffset).toHaveBeenCalledOnce(); + expect(observeFutureTimestampOffset).toHaveBeenCalledWith( + { + cacheNamespace: "urn", + useCase, + keyType: "user_id", + layer: CacheLayer.REMOTE, + }, + 1.25, + ); + expect(metrics.miss).toHaveBeenCalledOnce(); + }); + + it("serves a future-dated untracked frame without consulting the reader clock", async () => { + const cachedValue = { source: "redis" }; + const redis: DialCacheRedisClient = { + read: vi.fn(async () => ({ payload: JSON.stringify(cachedValue), createdAtMs: Number.MAX_SAFE_INTEGER })), + write: vi.fn(async () => undefined), + invalidate: vi.fn(async () => undefined), + }; + const observeFutureTimestampOffset = vi.fn(); + const metrics = metricsWithFutureTimestampObserver(observeFutureTimestampOffset); + const serializer: Serializer = { + dump: vi.fn((value) => JSON.stringify(value)), + load: vi.fn((value) => JSON.parse(Buffer.isBuffer(value) ? value.toString("utf8") : value)), + }; + const fallback = vi.fn(async () => ({ source: "fallback" })); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const getUser = dialcache.cached(fallback, { + keyType: "user_id", + useCase: "RedisUntrackedFutureFrame", + cacheKey: () => "123", + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + ramp: { [CacheLayer.REMOTE]: 100 }, + }), + serializer, + }); + const nowSpy = vi.spyOn(Date, "now").mockImplementation(() => { + throw new Error("untracked reads must not consult the reader clock"); + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual(cachedValue); + + expect(nowSpy).not.toHaveBeenCalled(); + expect(serializer.load).toHaveBeenCalledOnce(); + expect(serializer.dump).not.toHaveBeenCalled(); + expect(fallback).not.toHaveBeenCalled(); + expect(observeFutureTimestampOffset).not.toHaveBeenCalled(); + expect(metrics.miss).not.toHaveBeenCalled(); + }); + + it.each([ + Number.NaN, + Number.POSITIVE_INFINITY, + -1, + 1.5, + Number.MAX_SAFE_INTEGER + 1, + ])("treats invalid tracked frame timestamp %s as a miss without observing an offset", async (createdAtMs) => { + const redis: DialCacheRedisClient = { + read: vi.fn(async () => ({ payload: JSON.stringify({ source: "redis" }), createdAtMs })), + write: vi.fn(async () => undefined), + invalidate: vi.fn(async () => undefined), + }; + const observeFutureTimestampOffset = vi.fn(); + const metrics = metricsWithFutureTimestampObserver(observeFutureTimestampOffset); + const serializer: Serializer<{ readonly source: string }> = { + dump: vi.fn((value) => JSON.stringify(value)), + load: vi.fn(() => { + throw new Error("invalid tracked frames must not be deserialized"); + }), + }; + const fallback = vi.fn(async () => ({ source: "fallback" })); + const dialcache = new DialCache({ redis: { client: redis, readTimeoutMs: 1_000 }, metrics }); + const getUser = dialcache.cached(fallback, { + keyType: "user_id", + useCase: "RedisInvalidTrackedFrameTimestamp", + cacheKey: () => "123", + trackForInvalidation: true, + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + ramp: { [CacheLayer.REMOTE]: 100 }, + }), + serializer, + }); + const nowSpy = vi.spyOn(Date, "now"); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual({ source: "fallback" }); + + expect(nowSpy).not.toHaveBeenCalled(); + expect(serializer.load).not.toHaveBeenCalled(); + expect(fallback).toHaveBeenCalledOnce(); + expect(observeFutureTimestampOffset).not.toHaveBeenCalled(); + expect(metrics.miss).toHaveBeenCalledOnce(); + }); + + it("keeps a frame stamped exactly at the reader clock eligible", async () => { + const nowMs = 1_700_000_000_000; + vi.spyOn(Date, "now").mockReturnValue(nowMs); + const redis = new FakeRedis(); + const useCase = "RedisEqualTimestampFrame"; + const key = keyFor("123", useCase, true); + const cachedValue = { source: "redis" }; + redis.setRaw(`${key.urn}:dialcache-frame-v1`, encodeFrame(JSON.stringify(cachedValue), nowMs)); + redis.setRaw(`${key.prefix}#watermark`, "0"); + const observeFutureTimestampOffset = vi.fn(); + const metrics = metricsWithFutureTimestampObserver(observeFutureTimestampOffset); + const serializer: Serializer = { + dump: vi.fn((value) => JSON.stringify(value)), + load: vi.fn((value) => JSON.parse(Buffer.isBuffer(value) ? value.toString("utf8") : value)), + }; + const fallback = vi.fn(async () => ({ source: "fallback" })); + const dialcache = new DialCache({ + redis: { client: redis, readTimeoutMs: 1_000 }, + metrics, + }); + const getUser = dialcache.cached(fallback, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + trackForInvalidation: true, + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + ramp: { [CacheLayer.REMOTE]: 100 }, + }), + serializer, + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual(cachedValue); + + expect(serializer.load).toHaveBeenCalledOnce(); + expect(serializer.dump).not.toHaveBeenCalled(); + expect(fallback).not.toHaveBeenCalled(); + expect(observeFutureTimestampOffset).not.toHaveBeenCalled(); + expect(metrics.miss).not.toHaveBeenCalled(); + }); + + it.each(["throw", "reject"] as const)( + "fails open when the future-timestamp observer returns a %s failure", + async (failure) => { + const nowMs = 1_700_000_000_000; + vi.spyOn(Date, "now").mockReturnValue(nowMs); + const redis = new FakeRedis(); + const useCase = `RedisFutureTimestampObserver${failure}`; + const key = keyFor("123", useCase, true); + redis.setRaw( + `${key.urn}:dialcache-frame-v1`, + encodeFrame(JSON.stringify({ source: "redis" }), nowMs + 1), + ); + redis.setRaw(`${key.prefix}#watermark`, "0"); + const observerFailure = new Error("metrics transport failed"); + const observeFutureTimestampOffset = failure === "throw" + ? vi.fn(() => { + throw observerFailure; + }) + : vi.fn(() => Promise.reject(observerFailure)); + const metrics = metricsWithFutureTimestampObserver(observeFutureTimestampOffset); + const fallback = vi.fn(async () => ({ source: "fallback" })); + const dialcache = new DialCache({ + redis: { client: redis, readTimeoutMs: 1_000 }, + metrics, + }); + const getUser = dialcache.cached(fallback, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + trackForInvalidation: true, + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + ramp: { [CacheLayer.REMOTE]: 100 }, + }), + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual({ source: "fallback" }); + await Promise.resolve(); + + expect(observeFutureTimestampOffset).toHaveBeenCalledOnce(); + expect(fallback).toHaveBeenCalledOnce(); + }, + ); + it.each(["dialcache:", undefined])("rejects the removed Redis keyPrefix option value %s for untyped callers", (keyPrefix) => { const legacyRedisConfig = { client: new FakeRedis(), keyPrefix } as unknown as RedisConfig; @@ -424,7 +668,7 @@ describe("DialCache Redis TTL layer", () => { read: vi.fn(async () => { throw new DialCacheRedisPayloadEncodingError("Invalid DialCache Redis payload encoding"); }), - write: vi.fn(async () => true), + write: vi.fn(async () => {}), invalidate: vi.fn(async () => undefined), }; const logger = { debug: vi.fn(), warn: vi.fn(), error: vi.fn() }; diff --git a/test/dialcache-shadow-confirmation.test.ts b/test/dialcache-shadow-confirmation.test.ts index 0bd33b7..bba47b9 100644 --- a/test/dialcache-shadow-confirmation.test.ts +++ b/test/dialcache-shadow-confirmation.test.ts @@ -56,11 +56,12 @@ function deferred(): Deferred { type ReadStep = () => RedisCachePayload | null | Promise; const SCRIPTED_FRAME_CREATED_AT_MS = 1_700_000_000_000; +const MAX_TRACKED_REDIS_VALUE_TTL_MS = 60 * 60 * 1_000; class ScriptedRedis implements DialCacheRedisClient { readonly requests: RedisReadRequest[] = []; readonly contexts: Array = []; - readonly write = vi.fn(async (_request: RedisWriteRequest): Promise => true); + readonly write = vi.fn(async (_request: RedisWriteRequest): Promise => undefined); readonly invalidate = vi.fn(async (_request: RedisInvalidationRequest): Promise => undefined); frameCreatedAtMs = SCRIPTED_FRAME_CREATED_AT_MS; @@ -100,10 +101,16 @@ interface ShadowAgeEvent { readonly seconds: number; } +interface FutureTimestampEvent { + readonly labels: CacheMetricLabels; + readonly seconds: number; +} + class RecordingMetrics implements DialCacheMetricsAdapter { readonly ordinaryEvents: OrdinaryMetricEvent[] = []; readonly shadowEvents: ShadowValidationMetricLabels[] = []; readonly shadowAgeEvents: ShadowAgeEvent[] = []; + readonly futureTimestampEvents: FutureTimestampEvent[] = []; request(labels: CacheMetricLabels): void { this.record("request", labels); @@ -137,6 +144,10 @@ class RecordingMetrics implements DialCacheMetricsAdapter { this.shadowAgeEvents.push({ labels: { ...labels }, seconds }); } + observeFutureTimestampOffset(labels: CacheMetricLabels, seconds: number): void { + this.futureTimestampEvents.push({ labels: { ...labels }, seconds }); + } + observeGet(labels: CacheMetricLabels, _seconds: number): void { this.record("get", labels); } @@ -550,6 +561,119 @@ describe("DialCache Redis shadow confirmation", () => { expectTrackedReads(redis, 2); }); + it("retains a future-dated C1 for payload confirmation and attributes its offset to remote_shadow", async () => { + const nowMs = 1_700_000_000_000; + const nowSpy = vi.spyOn(Date, "now").mockReturnValue(nowMs); + try { + const payload = JSON.stringify({ id: "123", version: 1 }); + const redis = new ScriptedRedis([ + () => payload, + () => { + redis.frameCreatedAtMs = nowMs + 3_000; + return payload; + }, + ]); + redis.frameCreatedAtMs = nowMs - 1_000; + const metrics = new RecordingMetrics(); + const serializer: Serializer<{ readonly id: string; readonly version: number }> = { + dump: vi.fn((value) => JSON.stringify(value)), + load: vi.fn((value) => JSON.parse(Buffer.isBuffer(value) ? value.toString("utf8") : value)), + }; + const dialcache = createCache(redis, metrics); + const getUser = dialcache.cached(async () => ({ id: "123", version: 2 }), { + ...trackedOptions("ShadowFutureConfirmation", remoteConfig(100)), + cacheKey: () => "123", + serializer, + }); + + await dialcache.enable(async () => await getUser()); + await waitForShadowEvents(metrics, 1); + + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["mismatch"]); + expect(metrics.shadowAgeEvents).toEqual([ + { + labels: { + cacheNamespace: "urn", + useCase: "ShadowFutureConfirmation", + keyType: "user_id", + outcome: "mismatch", + }, + seconds: 1, + }, + ]); + expect(metrics.futureTimestampEvents).toEqual([ + { + labels: { + cacheNamespace: "urn", + useCase: "ShadowFutureConfirmation", + keyType: "user_id", + layer: REMOTE_SHADOW_CACHE_LAYER, + }, + seconds: 3, + }, + ]); + expect(metrics.ordinaryEvents.filter(({ name, labels }) => + name === "miss" && labels.layer === REMOTE_SHADOW_CACHE_LAYER + )).toHaveLength(0); + expect(serializer.load).toHaveBeenCalledTimes(2); + expect(serializer.dump).not.toHaveBeenCalled(); + expectTrackedReads(redis, 2); + } finally { + nowSpy.mockRestore(); + } + }); + + it("confirms the same C1 payload when the reader clock steps backward after accepting C0", async () => { + const nowMs = 1_700_000_000_000; + const nowSpy = vi.spyOn(Date, "now") + .mockReturnValueOnce(nowMs) + .mockReturnValue(nowMs - 2_000); + try { + const payload = JSON.stringify({ id: "123", version: 1 }); + const redis = new ScriptedRedis([() => payload, () => payload]); + redis.frameCreatedAtMs = nowMs - 1_000; + const metrics = new RecordingMetrics(); + const dialcache = createCache(redis, metrics); + const getUser = dialcache.cached(async () => ({ id: "123", version: 2 }), { + ...trackedOptions("ShadowConfirmationClockStepBack", remoteConfig(100)), + cacheKey: () => "123", + }); + + await dialcache.enable(async () => await getUser()); + await waitForShadowEvents(metrics, 1); + + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["mismatch"]); + expect(metrics.shadowAgeEvents).toEqual([ + { + labels: { + cacheNamespace: "urn", + useCase: "ShadowConfirmationClockStepBack", + keyType: "user_id", + outcome: "mismatch", + }, + seconds: 0, + }, + ]); + expect(metrics.futureTimestampEvents).toEqual([ + { + labels: { + cacheNamespace: "urn", + useCase: "ShadowConfirmationClockStepBack", + keyType: "user_id", + layer: REMOTE_SHADOW_CACHE_LAYER, + }, + seconds: 1, + }, + ]); + expect(metrics.ordinaryEvents.filter(({ name, labels }) => + name === "miss" && labels.layer === REMOTE_SHADOW_CACHE_LAYER + )).toHaveLength(0); + expectTrackedReads(redis, 2); + } finally { + nowSpy.mockRestore(); + } + }); + it("records the validated value age only for a confirmed mismatch verdict", async () => { const nowMs = 1_700_000_090_000; const nowSpy = vi.spyOn(Date, "now").mockReturnValue(nowMs); @@ -589,22 +713,27 @@ describe("DialCache Redis shadow confirmation", () => { } }); - it("skips the value-age observation when an out-of-contract client stamps a non-finite time", async () => { + it("treats a non-finite tracked dark frame timestamp as a miss without observing an offset", async () => { const payload = JSON.stringify({ id: "123", version: 1 }); const redis = new ScriptedRedis([() => payload]); redis.frameCreatedAtMs = Number.NaN; const metrics = new RecordingMetrics(); const dialcache = createCache(redis, metrics); const getUser = dialcache.cached(async () => ({ id: "123", version: 1 }), { - ...trackedOptions("ShadowValueAgeNonFinite", remoteConfig(100)), + ...trackedOptions("ShadowValueAgeNonFinite", remoteConfig(0)), cacheKey: () => "123", }); await dialcache.enable(async () => await getUser()); await waitForShadowEvents(metrics, 1); - expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["match"]); + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["filled"]); expect(metrics.shadowAgeEvents).toEqual([]); + expect(metrics.futureTimestampEvents).toEqual([]); + expect(redis.write).toHaveBeenCalledOnce(); + expect(metrics.ordinaryEvents.filter(({ name, labels }) => + name === "miss" && labels.layer === REMOTE_SHADOW_CACHE_LAYER + )).toHaveLength(1); }); it("does not log a mismatch candidate when C1 is superseded", async () => { @@ -689,6 +818,39 @@ describe("DialCache Redis shadow confirmation", () => { expectTrackedReads(redis, 2); }); + it("reports a tracked TTL clamp when a dark fill is actually dispatched", async () => { + const redis = new ScriptedRedis([() => null]); + const metrics = new RecordingMetrics(); + const config = new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 2 * MAX_TRACKED_REDIS_VALUE_TTL_MS / 1_000 }, + ramp: { [CacheLayer.REMOTE]: 0 }, + shadow: { ramp: 100 }, + }); + const dialcache = createCache(redis, metrics); + const getUser = dialcache.cached(async () => ({ id: "123" }), { + ...trackedOptions("ShadowDarkTrackedTtlClamp", config), + cacheKey: () => "123", + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toEqual({ id: "123" }); + await waitForShadowEvents(metrics, 1); + + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["filled"]); + expect(redis.write).toHaveBeenCalledWith(expect.objectContaining({ + cacheTtlMs: MAX_TRACKED_REDIS_VALUE_TTL_MS, + })); + expect(metrics.ordinaryEvents.filter(({ name }) => name === "error").map(({ labels }) => labels)).toEqual([ + { + cacheNamespace: "urn", + useCase: "ShadowDarkTrackedTtlClamp", + keyType: "user_id", + layer: REMOTE_SHADOW_CACHE_LAYER, + error: "tracked_ttl_clamped", + inFallback: false, + }, + ]); + }); + it("reports confirmation_error with its Redis work attributed to remote_shadow", async () => { const payload = JSON.stringify({ id: "123", version: 1 }); const redis = new ScriptedRedis([ @@ -781,6 +943,57 @@ describe("DialCache Redis shadow confirmation", () => { ).toHaveLength(1); }); + it("treats a future-dated dark C0 as a miss before deserialization and fills from SoT", async () => { + const nowMs = 1_700_000_000_000; + const nowSpy = vi.spyOn(Date, "now").mockReturnValue(nowMs); + try { + const redis = new ScriptedRedis([() => JSON.stringify({ id: "123", source: "redis" })]); + redis.frameCreatedAtMs = nowMs + 2_000; + const metrics = new RecordingMetrics(); + const sourceValue = { id: "123", source: "source" }; + const serializer: Serializer = { + dump: vi.fn((value) => JSON.stringify(value)), + load: vi.fn(() => { + throw new Error("future dark reads must not be deserialized"); + }), + }; + const source = vi.fn(async () => sourceValue); + const dialcache = createCache(redis, metrics); + const getUser = dialcache.cached(source, { + ...trackedOptions("ShadowDarkFutureFrame", remoteConfig(0)), + cacheKey: () => "123", + serializer, + }); + + await expect(dialcache.enable(async () => await getUser())).resolves.toBe(sourceValue); + await waitForShadowEvents(metrics, 1); + + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["filled"]); + expect(metrics.shadowAgeEvents).toEqual([]); + expect(metrics.futureTimestampEvents).toEqual([ + { + labels: { + cacheNamespace: "urn", + useCase: "ShadowDarkFutureFrame", + keyType: "user_id", + layer: REMOTE_SHADOW_CACHE_LAYER, + }, + seconds: 2, + }, + ]); + expect(serializer.load).not.toHaveBeenCalled(); + expect(serializer.dump).toHaveBeenCalledOnce(); + expect(source).toHaveBeenCalledOnce(); + expect(redis.write).toHaveBeenCalledOnce(); + expectTrackedReads(redis, 1); + expect(metrics.ordinaryEvents.filter(({ name, labels }) => + name === "miss" && labels.layer === REMOTE_SHADOW_CACHE_LAYER + )).toHaveLength(1); + } finally { + nowSpy.mockRestore(); + } + }); + it("unrefs ramp-zero shadow C0 and C1 Redis read-deadline timers", async () => { const cachedPayload = JSON.stringify({ id: "123", version: 1 }); const c0Started = deferred(); @@ -1029,7 +1242,7 @@ describe("DialCache Redis shadow confirmation", () => { cacheTtlMs: 60_000, value: JSON.stringify({ id: "123" }), })); - expect(Object.hasOwn(redis.write.mock.calls[0]?.[0] ?? {}, "watermarkKey")).toBe(tracked); + expect(Object.hasOwn(redis.write.mock.calls[0]?.[0] ?? {}, "watermarkKey")).toBe(false); expect(metrics.ordinaryEvents.filter(({ name, labels }) => name === "request" && labels.layer === REMOTE_SHADOW_CACHE_LAYER )).toHaveLength(1); @@ -1049,33 +1262,6 @@ describe("DialCache Redis shadow confirmation", () => { )).toHaveLength(1); }); - it("reports fill_blocked when tracked invalidation rejects a detached fill", async () => { - const redis = new ScriptedRedis([() => null]); - redis.write.mockImplementationOnce(async () => false); - const metrics = new RecordingMetrics(); - const sourceValue = { id: "123", version: 2 }; - const dialcache = createCache(redis, metrics); - const getUser = dialcache.cached(async () => sourceValue, { - ...trackedOptions("ShadowDarkFillBlocked", remoteConfig(0)), - cacheKey: () => "123", - }); - - await expect(dialcache.enable(async () => await getUser())).resolves.toBe(sourceValue); - await waitForShadowEvents(metrics, 1); - - expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["fill_blocked"]); - expect(redis.write).toHaveBeenCalledOnce(); - expect(redis.write).toHaveBeenCalledWith(expect.objectContaining({ - cacheTtlMs: 60_000, - watermarkKey: expect.any(String), - })); - expect(redis.invalidate).not.toHaveBeenCalled(); - expect(metrics.ordinaryEvents.filter(({ name, labels }) => - name === "error" - && labels.layer === REMOTE_SHADOW_CACHE_LAYER - )).toHaveLength(0); - }); - it("reports a detached serializer dump failure as fill_error with an exact remote_shadow error", async () => { const redis = new ScriptedRedis([() => null]); const metrics = new RecordingMetrics(); @@ -1212,7 +1398,7 @@ describe("DialCache Redis shadow confirmation", () => { const dumpStarted = deferred(); const dumpGate = deferred(); const writeStarted = deferred(); - const writeGate = deferred(); + const writeGate = deferred(); const redis = new ScriptedRedis([() => null]); redis.write.mockImplementationOnce(async () => { writeStarted.resolve(undefined); @@ -1249,12 +1435,12 @@ describe("DialCache Redis shadow confirmation", () => { await writeStarted.promise; expect(metrics.shadowEvents).toHaveLength(0); - writeGate.resolve(true); + writeGate.resolve(undefined); await waitForShadowEvents(metrics, 1); expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["filled"]); } finally { dumpGate.resolve(undefined); - writeGate.resolve(true); + writeGate.resolve(undefined); } }); @@ -1322,8 +1508,8 @@ describe("DialCache Redis shadow confirmation", () => { expect(redis.write).toHaveBeenCalledOnce(); expect(redis.write).toHaveBeenCalledWith(expect.objectContaining({ cacheTtlMs: 17_000, - watermarkKey: expect.any(String), })); + expect(Object.hasOwn(redis.write.mock.calls[0]?.[0] ?? {}, "watermarkKey")).toBe(false); }); it("reports a dark Redis error with remote_shadow read telemetry and never writes", async () => { @@ -1725,17 +1911,12 @@ describe("DialCache Redis shadow confirmation", () => { expectTrackedReads(redis, 2, { singleWatermark: false }); }); - it.each([ - { name: "successful", result: true }, - { name: "watermark-blocked", result: false }, - ])("retains capacity after an overall timeout until an already-dispatched $name write settles", async ({ - result, - }) => { + it("retains capacity after an overall timeout until an already-dispatched write settles", async () => { vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); let nowMs = 0; const performanceSpy = vi.spyOn(performance, "now").mockImplementation(() => nowMs); const writeStarted = deferred(); - const writeGate = deferred(); + const writeGate = deferred(); const redis = new ScriptedRedis([ () => null, () => JSON.stringify({ id: "b" }), @@ -1765,7 +1946,7 @@ describe("DialCache Redis shadow confirmation", () => { expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["timeout", "dropped"]); expect(redis.requests).toHaveLength(1); - writeGate.resolve(result); + writeGate.resolve(undefined); await nextImmediate(); await expect(dialcache.enable(async () => await getUser("b"))).resolves.toEqual({ id: "b" }); @@ -1778,7 +1959,7 @@ describe("DialCache Redis shadow confirmation", () => { expect(redis.write).toHaveBeenCalledOnce(); expectTrackedReads(redis, 2, { singleWatermark: false }); } finally { - writeGate.resolve(result); + writeGate.resolve(undefined); performanceSpy.mockRestore(); vi.useRealTimers(); } @@ -1804,8 +1985,13 @@ describe("DialCache Redis shadow confirmation", () => { load: vi.fn(async (payload) => JSON.parse(payload.toString()) as { readonly id: string }), }; const dialcache = createCache(redis, metrics, { shadowMaxInFlight: 1 }); + const config = new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 2 * MAX_TRACKED_REDIS_VALUE_TTL_MS / 1_000 }, + ramp: { [CacheLayer.REMOTE]: 0 }, + shadow: { ramp: 100 }, + }); const getUser = dialcache.cached(async (id: string) => ({ id }), { - ...trackedOptions("ShadowDarkDumpTimeoutRetention", remoteConfig(0)), + ...trackedOptions("ShadowDarkDumpTimeoutRetention", config), cacheKey: (id) => id, fallbackTimeoutMs: 50, serializer, @@ -1827,6 +2013,9 @@ describe("DialCache Redis shadow confirmation", () => { dumpGate.resolve(undefined); await nextImmediate(); expect(redis.write).not.toHaveBeenCalled(); + expect(metrics.ordinaryEvents.filter(({ name, labels }) => + name === "error" && labels.error === "tracked_ttl_clamped" + )).toHaveLength(0); await expect(dialcache.enable(async () => await getUser("b"))).resolves.toEqual({ id: "b" }); await waitForShadowEvents(metrics, 3); diff --git a/test/dialcache-shadow-validation.test.ts b/test/dialcache-shadow-validation.test.ts index 0eed650..dfe1d30 100644 --- a/test/dialcache-shadow-validation.test.ts +++ b/test/dialcache-shadow-validation.test.ts @@ -25,9 +25,15 @@ interface ShadowAgeEvent { readonly seconds: number; } +interface FutureTimestampEvent { + readonly labels: CacheMetricLabels; + readonly seconds: number; +} + class RecordingMetrics implements DialCacheMetricsAdapter { readonly shadowEvents: ShadowValidationMetricLabels[] = []; readonly shadowAgeEvents: ShadowAgeEvent[] = []; + readonly futureTimestampEvents: FutureTimestampEvent[] = []; readonly errorEvents: ErrorMetricLabels[] = []; request(_labels: CacheMetricLabels): void {} @@ -48,6 +54,10 @@ class RecordingMetrics implements DialCacheMetricsAdapter { this.shadowAgeEvents.push({ labels: { ...labels }, seconds }); } + observeFutureTimestampOffset(labels: CacheMetricLabels, seconds: number): void { + this.futureTimestampEvents.push({ labels: { ...labels }, seconds }); + } + observeGet(_labels: CacheMetricLabels, _seconds: number): void {} observeFallback(_labels: CacheMetricLabels, _seconds: number): void {} observeSerialization(_labels: SerializationMetricLabels, _seconds: number): void {} @@ -341,37 +351,78 @@ describe("DialCache Redis shadow validation", () => { } }); - it("clamps a future-stamped frame to a zero value age instead of a negative one", async () => { + it("clamps value age to zero when the reader clock steps backward after accepting the frame", async () => { const nowMs = 1_700_000_000_000; const nowSpy = vi.spyOn(Date, "now").mockReturnValue(nowMs); try { const redis = new FakeRedis(); const metrics = new RecordingMetrics(); - const useCase = "ShadowValueAgeClamped"; + const useCase = "ShadowValueAgeClockRollback"; const cachedValue = { id: "123" }; seedRedis(redis, { id: "123", useCase, payload: JSON.stringify(cachedValue), - createdAtMs: nowMs + 60_000, + createdAtMs: nowMs, }); + const sourceStarted = deferred(); + const sourceGate = deferred(); const dialcache = createShadowCache(redis, metrics); - const getUser = dialcache.cached(async () => cachedValue, { + const getUser = dialcache.cached(async () => { + sourceStarted.resolve(undefined); + return await sourceGate.promise; + }, { ...trackedRemoteDefaults(useCase), cacheKey: () => "123", }); expect(await dialcache.enable(async () => await getUser())).toEqual(cachedValue); + await sourceStarted.promise; + nowSpy.mockReturnValue(nowMs - 60_000); + sourceGate.resolve(cachedValue); await waitForShadowEvents(metrics, 1); expect(metrics.shadowEvents[0]?.outcome).toBe("match"); expect(metrics.shadowAgeEvents).toHaveLength(1); expect(metrics.shadowAgeEvents[0]?.seconds).toBe(0); + expect(metrics.futureTimestampEvents).toEqual([]); } finally { nowSpy.mockRestore(); } }); + it("skips a non-finite value-age observation from an untracked custom client", async () => { + const redis = new FakeRedis(); + const metrics = new RecordingMetrics(); + const useCase = "ShadowNonFiniteValueAge"; + const cachedValue = { id: "123", version: 1 }; + seedRedis(redis, { + id: "123", + useCase, + payload: JSON.stringify(cachedValue), + tracked: false, + }); + const originalRead = redis.read.bind(redis); + vi.spyOn(redis, "read").mockImplementation(async (request) => { + const frame = await originalRead(request); + return frame === null ? null : { ...frame, createdAtMs: Number.POSITIVE_INFINITY }; + }); + const dialcache = createShadowCache(redis, metrics); + const getUser = dialcache.cached(async () => cachedValue, { + keyType: "user_id", + useCase, + cacheKey: () => "123", + defaultConfig: remoteOnly(100), + }); + + expect(await dialcache.enable(async () => await getUser())).toEqual(cachedValue); + await waitForShadowEvents(metrics, 1); + + expect(metrics.shadowEvents[0]?.outcome).toBe("match"); + expect(metrics.shadowAgeEvents).toEqual([]); + expect(metrics.futureTimestampEvents).toEqual([]); + }); + it("re-deserializes the retained payload instead of comparing a caller-mutated hit", async () => { const redis = new FakeRedis(); const metrics = new RecordingMetrics(); diff --git a/test/duration.test.ts b/test/duration.test.ts index 9724f85..7a1f317 100644 --- a/test/duration.test.ts +++ b/test/duration.test.ts @@ -6,7 +6,9 @@ import { isSupportedCacheTtlSec, MAX_CACHE_TTL_SEC, MAX_SUPPORTED_DURATION_MS, + MAX_TRACKED_REDIS_VALUE_TTL_MS, } from "../src/internal/duration.js"; +import { MIN_WATERMARK_TTL_MS } from "../src/internal/redis-scripts.js"; describe("DialCache supported durations", () => { it("uses one fixed 365-day ceiling for TTLs and invalidation buffers", () => { @@ -17,6 +19,13 @@ describe("DialCache supported durations", () => { expect(() => assertSupportedFutureBufferMs(MAX_SUPPORTED_DURATION_MS)).not.toThrow(); }); + it("pins tracked Redis lifetimes as protocol-cutover constants", () => { + // Increasing the value cap is not made rolling-safe by increasing the + // current floor: already-deployed invalidators retain their compiled floor. + expect(MAX_TRACKED_REDIS_VALUE_TTL_MS).toBe(60 * 60 * 1_000); + expect(MIN_WATERMARK_TTL_MS).toBe(2 * 60 * 60 * 1_000); + }); + it.each([0, MAX_CACHE_TTL_SEC + 1, Number.MAX_SAFE_INTEGER])( "rejects unsupported cache TTL %s", (ttlSec) => { diff --git a/test/fake-redis.ts b/test/fake-redis.ts index 4633b99..2a004f2 100644 --- a/test/fake-redis.ts +++ b/test/fake-redis.ts @@ -1,12 +1,14 @@ import type { DecodedRedisFrame, DialCacheRedisClient, - RedisCachePayload, RedisInvalidationRequest, RedisReadRequest, RedisWriteRequest, } from "../src/index.js"; +import { MAX_TRACKED_REDIS_VALUE_TTL_MS } from "../src/internal/duration.js"; +import { MIN_WATERMARK_TTL_MS } from "../src/internal/redis-scripts.js"; import { DialCacheRedisPayloadEncodingError } from "../src/redis-client.js"; +import { ceilSupportedCacheTtlMs, encodeRedisFrame } from "../src/redis-protocol.js"; const FRAME_VERSION = 1; const ENCODING_OFFSET = 9; @@ -41,29 +43,22 @@ export class FakeRedis implements DialCacheRedisClient { async write({ valueKey, - watermarkKey, cacheTtlMs, value, - }: RedisWriteRequest): Promise { + }: RedisWriteRequest): Promise { + const validatedTtlMs = ceilSupportedCacheTtlMs(cacheTtlMs); + const createdAtMs = Date.now(); + const frame = encodeRedisFrame(value, createdAtMs); this.setCalls += 1; this.throwIfWriteFails(); - if (watermarkKey !== undefined) { - const watermark = this.readWatermark(watermarkKey) ?? 0; - if (watermark >= Date.now()) { - return false; - } - this.storeFrame(valueKey, cacheTtlMs, value); - const currentTtlMs = this.remainingTtlMs(watermarkKey); - const desiredTtlMs = Math.max(currentTtlMs, cacheTtlMs + WATERMARK_TTL_MARGIN_MS); - this.storeWatermark(watermarkKey, watermark, desiredTtlMs); - return true; - } - - this.storeFrame(valueKey, cacheTtlMs, value); - return true; + this.values.set(valueKey, { + value: frame, + expiresAtMs: createdAtMs + validatedTtlMs, + }); } async invalidate({ watermarkKey, futureBufferMs }: RedisInvalidationRequest): Promise { + const invalidatedAtMs = Date.now(); this.setCalls += 1; this.throwIfWriteFails(); let current = 0; @@ -72,12 +67,12 @@ export class FakeRedis implements DialCacheRedisClient { } catch { current = 0; } - const watermark = Math.max(current, Date.now() + futureBufferMs); + const watermark = Math.max(current, invalidatedAtMs + futureBufferMs); const currentTtlMs = this.remainingTtlMs(watermarkKey); const desiredTtlMs = Math.max( currentTtlMs, - futureBufferMs + WATERMARK_TTL_MARGIN_MS, - watermark - Date.now() + WATERMARK_TTL_MARGIN_MS, + MIN_WATERMARK_TTL_MS, + watermark - invalidatedAtMs + MAX_TRACKED_REDIS_VALUE_TTL_MS + WATERMARK_TTL_MARGIN_MS, ); this.storeWatermark(watermarkKey, watermark, desiredTtlMs); } @@ -137,7 +132,7 @@ export class FakeRedis implements DialCacheRedisClient { } catch { return null; } - if (watermark === null || createdAtMs <= watermark) { + if (createdAtMs <= (watermark ?? 0)) { return null; } } @@ -152,22 +147,8 @@ export class FakeRedis implements DialCacheRedisClient { throw new DialCacheRedisPayloadEncodingError("Invalid DialCache Redis payload encoding"); } - private storeFrame(key: string, ttlMs: number, payload: RedisCachePayload): void { - const timestamp = Buffer.alloc(8); - timestamp.writeBigUInt64BE(BigInt(Date.now())); - this.values.set(key, { - value: Buffer.concat([ - Buffer.from([FRAME_VERSION]), - timestamp, - Buffer.from([Buffer.isBuffer(payload) ? 1 : 0]), - Buffer.from(payload), - ]), - expiresAtMs: Date.now() + ttlMs, - }); - } - private storeWatermark(key: string, watermark: number, ttlMs: number): void { - this.values.set(key, { value: Buffer.from(String(Math.floor(watermark))), expiresAtMs: Date.now() + ttlMs }); + this.values.set(key, { value: Buffer.from(String(Math.ceil(watermark))), expiresAtMs: Date.now() + ttlMs }); } private readWatermark(key: string): number | null { @@ -176,14 +157,14 @@ export class FakeRedis implements DialCacheRedisClient { return null; } const text = raw.toString("utf8"); - if (!/^\d+(?:\.\d+)?$/.test(text)) { + if (!/^\d+$/.test(text)) { throw new Error("Invalid DialCache watermark"); } - const legacy = Number(text); - if (!Number.isFinite(legacy) || legacy < 0) { + const watermark = Number(text); + if (watermark > Number.MAX_SAFE_INTEGER) { throw new Error("Invalid DialCache watermark"); } - return legacy; + return watermark; } private readRaw(key: string): Buffer | null { diff --git a/test/node-redis.test.ts b/test/node-redis.test.ts index 258e3aa..3867fee 100644 --- a/test/node-redis.test.ts +++ b/test/node-redis.test.ts @@ -1,16 +1,19 @@ -import { describe, expect, it, vi } from "vitest"; +import { createHash } from "node:crypto"; + +import { afterEach, describe, expect, it, vi } from "vitest"; import { CacheLayer, DialCache, DialCacheKeyConfig, - DialCacheRedisPlaceholderLostError, DialCacheRedisProtocolError, } from "../src/index.js"; -import { createNodeRedisDialCacheClient, dialcacheRedisScripts } from "../src/node-redis.js"; +import { createNodeRedisDialCacheClient } from "../src/node-redis.js"; import { INVALIDATE_CACHE_SCRIPT } from "../src/redis-protocol.js"; -const INVALID_WRITE_REPLIES: readonly unknown[] = [ +const INVALID_INVALIDATION_REPLIES: readonly unknown[] = [ + 0, + 2, -1, 3, 0.5, @@ -23,15 +26,13 @@ const INVALID_WRITE_REPLIES: readonly unknown[] = [ null, undefined, ]; -const INVALID_INVALIDATION_REPLIES: readonly unknown[] = [0, 2, ...INVALID_WRITE_REPLIES]; interface FakeReplies { readonly get?: unknown; readonly mGet?: unknown; readonly set?: unknown; readonly eval?: unknown; - readonly stamp?: unknown; - readonly invalidate?: unknown; + readonly evalSha?: unknown; } function fakeClient(replies: FakeReplies = {}) { @@ -46,10 +47,11 @@ function fakeClient(replies: FakeReplies = {}) { if (args[0] === "EVAL") { return Object.hasOwn(replies, "eval") ? replies.eval : 1; } + if (args[0] === "EVALSHA") { + return Object.hasOwn(replies, "evalSha") ? replies.evalSha : 1; + } return Object.hasOwn(replies, "mGet") ? replies.mGet : [null, null]; }), - dialcacheWriteTrackedStamp: vi.fn(async () => Object.hasOwn(replies, "stamp") ? replies.stamp : 1), - dialcacheInvalidate: vi.fn(async () => Object.hasOwn(replies, "invalidate") ? replies.invalidate : 1), }; } @@ -83,47 +85,16 @@ async function expectProtocolError(operation: Promise, message: string) } describe("node-redis adapter", () => { - it("provides the expected arguments for every bundled mutation script", () => { - const nonce = Buffer.from("01234567"); - expect(Object.keys(dialcacheRedisScripts)).toEqual([ - "dialcacheWriteTrackedStamp", - "dialcacheInvalidate", - ]); - expect( - dialcacheRedisScripts.dialcacheWriteTrackedStamp.transformArguments( - "tracked:{id}:value", - "tracked:{id}:watermark", - 1_000, - nonce, - ), - ).toEqual(["tracked:{id}:value", "tracked:{id}:watermark", "1000", nonce]); - expect( - dialcacheRedisScripts.dialcacheInvalidate.transformArguments("tracked:{id}:watermark", 50), - ).toEqual(["tracked:{id}:watermark", "50"]); + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("keeps invalidation independent of the Redis server clock", () => { + expect(INVALIDATE_CACHE_SCRIPT).not.toMatch(/redis\.call\(["']TIME["']\)/); }); - it("rejects clients constructed without the DialCache script registrations", () => { - expect( - () => createNodeRedisDialCacheClient({ get: vi.fn(), sendCommand: vi.fn() } as never), - ).toThrow(TypeError); - expect( - () => createNodeRedisDialCacheClient({ get: vi.fn(), sendCommand: vi.fn() } as never), - ).toThrow("requires a client created with scripts: dialcacheRedisScripts"); - // Partial registration must fail just as loudly as none. - expect( - () => createNodeRedisDialCacheClient({ - get: vi.fn(), - sendCommand: vi.fn(), - dialcacheWriteTrackedStamp: vi.fn(), - } as never), - ).toThrow(TypeError); - expect( - () => createNodeRedisDialCacheClient({ - get: vi.fn(), - sendCommand: vi.fn(), - dialcacheInvalidate: vi.fn(), - } as never), - ).toThrow(TypeError); + it("accepts an ordinary node-redis client without custom script registrations", () => { + expect(() => createNodeRedisDialCacheClient(fakeClient() as never)).not.toThrow(); }); it("accepts the exact write and invalidation reply domains", async () => { @@ -131,8 +102,7 @@ describe("node-redis adapter", () => { get: encodeFrame("plain"), mGet: [encodeFrame(Buffer.from([0, 0xff]), { createdAtMs: 2 }), Buffer.from("1")], set: "OK", - stamp: 0, - invalidate: 1, + evalSha: 1, }); const adapter = createNodeRedisDialCacheClient(client as never); @@ -145,30 +115,27 @@ describe("node-redis adapter", () => { ).resolves.toEqual({ payload: Buffer.from([0, 0xff]), createdAtMs: 2 }); await expect( adapter.write({ valueKey: "plain:value", cacheTtlMs: 1_000, value: "plain" }), - ).resolves.toBe(true); + ).resolves.toBeUndefined(); await expect( adapter.write({ valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", cacheTtlMs: 1_000, value: "tracked", }), - ).resolves.toBe(false); + ).resolves.toBeUndefined(); await expect( adapter.invalidate({ watermarkKey: "tracked:{id}:watermark", futureBufferMs: 50 }), ).resolves.toBeUndefined(); }); - it("writes untracked frames with one native SET", async () => { + it("writes complete frames with one native SET", async () => { + vi.spyOn(Date, "now").mockReturnValue(1_234); const client = fakeClient(); const adapter = createNodeRedisDialCacheClient(client as never); - const before = Date.now(); await expect( adapter.write({ valueKey: "plain:value", cacheTtlMs: 1_000, value: "plain" }), - ).resolves.toBe(true); - const after = Date.now(); + ).resolves.toBeUndefined(); - expect(client.dialcacheWriteTrackedStamp).not.toHaveBeenCalled(); expect(client.sendCommand).toHaveBeenCalledTimes(1); const [args, options] = client.sendCommand.mock.calls[0] as [Array, unknown]; expect(args[0]).toBe("SET"); @@ -179,86 +146,34 @@ describe("node-redis adapter", () => { expect(frame[0]).toBe(1); expect(frame[9]).toBe(0); expect(frame.subarray(10).toString("utf8")).toBe("plain"); - const createdAtMs = Number(frame.readBigUInt64BE(1)); - expect(createdAtMs).toBeGreaterThanOrEqual(before); - expect(createdAtMs).toBeLessThanOrEqual(after); + expect(Number(frame.readBigUInt64BE(1))).toBe(1_234); expect(options).toMatchObject({ returnBuffers: true }); }); - it("pairs a zero-stamped placeholder SET with the stamp script in issue order", async () => { - const order: string[] = []; + it("writes binary values as complete frames with one client-clock sample", async () => { + const now = vi.spyOn(Date, "now").mockReturnValue(1_234); const client = fakeClient(); - client.sendCommand.mockImplementation(async () => { - order.push("set"); - return "OK"; - }); - client.dialcacheWriteTrackedStamp.mockImplementation(async () => { - order.push("stamp"); - return 1; - }); const binary = Buffer.from([0, 0xff]); const adapter = createNodeRedisDialCacheClient(client as never); await expect(adapter.write({ valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", cacheTtlMs: 2_000, value: binary, - })).resolves.toBe(true); + })).resolves.toBeUndefined(); - expect(order).toEqual(["set", "stamp"]); + expect(client.sendCommand).toHaveBeenCalledTimes(1); const [args] = client.sendCommand.mock.calls[0] as [Array]; expect(args[0]).toBe("SET"); expect(args[1]).toBe("tracked:{id}:value"); expect(args[3]).toBe("PX"); expect(args[4]).toBe("2000"); const frame = args[2] as Buffer; - expect(frame[0]).toBe(0); + expect(frame[0]).toBe(1); expect(frame[9]).toBe(1); expect(frame.subarray(10)).toEqual(binary); - // The stamp must carry the exact nonce its paired placeholder was minted with. - expect(client.dialcacheWriteTrackedStamp).toHaveBeenCalledWith( - "tracked:{id}:value", - "tracked:{id}:watermark", - 2_000, - frame.subarray(1, 9), - ); - }); - - it("fails a tracked write whose placeholder was lost before the stamp", async () => { - const adapter = createNodeRedisDialCacheClient(fakeClient({ stamp: 2 }) as never); - const write = adapter.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", - cacheTtlMs: 1_000, - value: "tracked", - }); - await expect(write).rejects.toThrow("DialCache tracked write lost its placeholder before the stamp"); - await expect(write).rejects.toBeInstanceOf(DialCacheRedisPlaceholderLostError); - }); - - it("issues the stamp before the placeholder SET settles", async () => { - const client = fakeClient(); - let resolveSet: ((value: string) => void) | undefined; - client.sendCommand.mockImplementationOnce( - async () => await new Promise((resolve) => { - resolveSet = resolve; - }), - ); - const adapter = createNodeRedisDialCacheClient(client as never); - - const write = adapter.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", - cacheTtlMs: 1_000, - value: "tracked", - }); - // The stamp must already be issued while the SET is still unsettled: an - // await between the pair would leave it uncalled here and hang the write. - expect(client.dialcacheWriteTrackedStamp).toHaveBeenCalledTimes(1); - - resolveSet?.("OK"); - await expect(write).resolves.toBe(true); + expect(Number(frame.readBigUInt64BE(1))).toBe(1_234); + expect(now).toHaveBeenCalledTimes(1); }); it("routes cluster write SETs by the value key", async () => { @@ -267,10 +182,9 @@ describe("node-redis adapter", () => { await expect(adapter.write({ valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", cacheTtlMs: 1_000, value: "tracked", - })).resolves.toBe(true); + })).resolves.toBeUndefined(); const [firstKey, isReadonly, args] = client.sendCommand.mock.calls[0] as [string, boolean, Array]; expect(firstKey).toBe("tracked:{id}:value"); @@ -283,7 +197,7 @@ describe("node-redis adapter", () => { await expect( createNodeRedisDialCacheClient(fakeClient({ set: Buffer.from("OK") }) as never) .write({ valueKey: "plain:value", cacheTtlMs: 1_000, value: "plain" }), - ).resolves.toBe(true); + ).resolves.toBeUndefined(); for (const reply of ["QUEUED", null, 1, undefined, Buffer.from("NO")]) { const untracked = createNodeRedisDialCacheClient(fakeClient({ set: reply }) as never); @@ -292,16 +206,6 @@ describe("node-redis adapter", () => { "Invalid DialCache Redis SET reply; expected OK", ); - const tracked = createNodeRedisDialCacheClient(fakeClient({ set: reply }) as never); - await expectProtocolError( - Promise.resolve(tracked.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", - cacheTtlMs: 1_000, - value: "tracked", - })), - "Invalid DialCache Redis SET reply; expected OK", - ); } }); @@ -313,35 +217,43 @@ describe("node-redis adapter", () => { await expect( adapter.write({ valueKey: "plain:value", cacheTtlMs, value: "plain" }), ).rejects.toThrow(RangeError); - await expect( - adapter.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", - cacheTtlMs, - value: "tracked", - }), - ).rejects.toThrow(RangeError); } expect(client.sendCommand).not.toHaveBeenCalled(); - expect(client.dialcacheWriteTrackedStamp).not.toHaveBeenCalled(); await adapter.write({ valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", cacheTtlMs: 1_000.1, value: "tracked", }); const [args] = client.sendCommand.mock.calls[0] as [Array]; expect(args[4]).toBe("1001"); - expect(client.dialcacheWriteTrackedStamp).toHaveBeenCalledWith( - "tracked:{id}:value", - "tracked:{id}:watermark", - 1_001, - expect.any(Buffer), - ); }); - it("surfaces a SET failure as the write error even when the stamp settled", async () => { + it("rejects invalid application timestamps before dispatching mutations", async () => { + const now = vi.spyOn(Date, "now"); + const client = fakeClient(); + const adapter = createNodeRedisDialCacheClient(client as never); + + for (const timestampMs of [ + -1, + 1.5, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.MAX_SAFE_INTEGER + 1, + ]) { + now.mockReturnValue(timestampMs); + await expect( + adapter.write({ valueKey: "plain:value", cacheTtlMs: 1_000, value: "plain" }), + ).rejects.toThrow(RangeError); + await expect( + adapter.invalidate({ watermarkKey: "tracked:{id}:watermark", futureBufferMs: 50 }), + ).rejects.toThrow(RangeError); + } + + expect(client.sendCommand).not.toHaveBeenCalled(); + }); + + it("surfaces a SET failure as the write error", async () => { const failure = new Error("OOM command not allowed when used memory > 'maxmemory'."); const client = fakeClient(); client.sendCommand.mockRejectedValueOnce(failure); @@ -349,36 +261,9 @@ describe("node-redis adapter", () => { await expect(adapter.write({ valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", cacheTtlMs: 1_000, value: "tracked", })).rejects.toBe(failure); - expect(client.dialcacheWriteTrackedStamp).toHaveBeenCalledTimes(1); - - const stampFailure = new Error("ERR invalid DialCache watermark"); - const stampClient = fakeClient(); - stampClient.dialcacheWriteTrackedStamp.mockRejectedValueOnce(stampFailure); - const stampAdapter = createNodeRedisDialCacheClient(stampClient as never); - await expect(stampAdapter.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", - cacheTtlMs: 1_000, - value: "tracked", - })).rejects.toBe(stampFailure); - - // A bad SET reply also wins over a failing stamp, matching the contract. - const combinedClient = fakeClient({ set: "QUEUED" }); - combinedClient.dialcacheWriteTrackedStamp.mockRejectedValueOnce(new Error("ERR stamp")); - const combinedAdapter = createNodeRedisDialCacheClient(combinedClient as never); - await expectProtocolError( - Promise.resolve(combinedAdapter.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", - cacheTtlMs: 1_000, - value: "tracked", - })), - "Invalid DialCache Redis SET reply; expected OK", - ); }); it("passes the cooperative read signal through node-redis command options", async () => { @@ -479,24 +364,10 @@ describe("node-redis adapter", () => { }); it("rejects every out-of-domain reply returned by a node-redis client", async () => { - const writeMessage = "Invalid DialCache Redis write reply; expected integer 0, 1, or 2"; const invalidationMessage = "Invalid DialCache Redis invalidate reply; expected integer 1"; - for (const reply of INVALID_WRITE_REPLIES) { - const tracked = createNodeRedisDialCacheClient(fakeClient({ stamp: reply }) as never); - await expectProtocolError( - Promise.resolve(tracked.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", - cacheTtlMs: 1_000, - value: "tracked", - })), - writeMessage, - ); - } - for (const reply of INVALID_INVALIDATION_REPLIES) { - const adapter = createNodeRedisDialCacheClient(fakeClient({ invalidate: reply }) as never); + const adapter = createNodeRedisDialCacheClient(fakeClient({ evalSha: reply }) as never); await expectProtocolError( Promise.resolve(adapter.invalidate({ watermarkKey: "tracked:{id}:watermark", @@ -507,7 +378,8 @@ describe("node-redis adapter", () => { } }); - it("dispatches invalidation once and sends no EVAL when the registered script resolves", async () => { + it("dispatches invalidation once with EVALSHA by the source digest", async () => { + const now = vi.spyOn(Date, "now").mockReturnValue(1_234); const client = fakeClient(); const adapter = createNodeRedisDialCacheClient(client as never); @@ -515,13 +387,25 @@ describe("node-redis adapter", () => { adapter.invalidate({ watermarkKey: "tracked:{id}:watermark", futureBufferMs: 50 }), ).resolves.toBeUndefined(); - expect(client.dialcacheInvalidate).toHaveBeenCalledTimes(1); - expect(client.sendCommand).not.toHaveBeenCalled(); + expect(client.sendCommand).toHaveBeenCalledTimes(1); + const [args, options] = client.sendCommand.mock.calls[0] as [Array, object]; + expect(args).toEqual([ + "EVALSHA", + createHash("sha1").update(INVALIDATE_CACHE_SCRIPT).digest("hex"), + "1", + "tracked:{id}:watermark", + "50", + "1234", + ]); + expect(now).toHaveBeenCalledTimes(1); + expect(options).toMatchObject({ returnBuffers: true }); + expect(Object.keys(options)).toEqual(["returnBuffers"]); }); - it("retries a rejected invalidation dispatch once with EVAL by source", async () => { + it("retries a rejected EVALSHA once with EVAL by source", async () => { + const now = vi.spyOn(Date, "now").mockReturnValue(1_234); const client = fakeClient(); - client.dialcacheInvalidate.mockRejectedValueOnce( + client.sendCommand.mockRejectedValueOnce( new Error("NOPERM this user has no permissions to run the 'evalsha' command"), ); const adapter = createNodeRedisDialCacheClient(client as never); @@ -530,31 +414,39 @@ describe("node-redis adapter", () => { adapter.invalidate({ watermarkKey: "tracked:{id}:watermark", futureBufferMs: 50 }), ).resolves.toBeUndefined(); - expect(client.dialcacheInvalidate).toHaveBeenCalledTimes(1); - expect(client.sendCommand).toHaveBeenCalledTimes(1); - const [args, options] = client.sendCommand.mock.calls[0] as [Array, object]; + expect(client.sendCommand).toHaveBeenCalledTimes(2); + const [firstArgs] = client.sendCommand.mock.calls[0] as [Array]; + const [args, options] = client.sendCommand.mock.calls[1] as [Array, object]; + expect(firstArgs[0]).toBe("EVALSHA"); expect(args).toEqual([ "EVAL", INVALIDATE_CACHE_SCRIPT, "1", "tracked:{id}:watermark", "50", + "1234", ]); + expect(now).toHaveBeenCalledTimes(1); expect(options).toMatchObject({ returnBuffers: true }); expect(Object.keys(options)).toEqual(["returnBuffers"]); }); it("routes the invalidation EVAL retry through the cluster keyed overload", async () => { const client = fakeCluster(); - client.dialcacheInvalidate.mockRejectedValueOnce(new Error("NOPERM evalsha denied")); + client.sendCommand.mockRejectedValueOnce(new Error("NOPERM evalsha denied")); const adapter = createNodeRedisDialCacheClient(client as never); await expect( adapter.invalidate({ watermarkKey: "tracked:{id}:watermark", futureBufferMs: 50 }), ).resolves.toBeUndefined(); - expect(client.sendCommand).toHaveBeenCalledTimes(1); - const [firstKey, isReadonly, args, options] = client.sendCommand.mock.calls[0] as [ + expect(client.sendCommand).toHaveBeenCalledTimes(2); + const [firstKey, isReadonly, firstArgs] = client.sendCommand.mock.calls[0] as [ + string, + boolean, + Array, + ]; + const [retryKey, retryIsReadonly, args, options] = client.sendCommand.mock.calls[1] as [ string, boolean, Array, @@ -562,6 +454,9 @@ describe("node-redis adapter", () => { ]; expect(firstKey).toBe("tracked:{id}:watermark"); expect(isReadonly).toBe(false); + expect(firstArgs[0]).toBe("EVALSHA"); + expect(retryKey).toBe("tracked:{id}:watermark"); + expect(retryIsReadonly).toBe(false); expect(args[0]).toBe("EVAL"); expect(options).toMatchObject({ returnBuffers: true }); }); @@ -570,7 +465,7 @@ describe("node-redis adapter", () => { const client = fakeClient(); const original = new Error("NOPERM evalsha denied"); const retryFailure = new Error("NOPERM eval denied"); - client.dialcacheInvalidate.mockRejectedValueOnce(original); + client.sendCommand.mockRejectedValueOnce(original); client.sendCommand.mockRejectedValueOnce(retryFailure); const adapter = createNodeRedisDialCacheClient(client as never); @@ -579,7 +474,7 @@ describe("node-redis adapter", () => { ).rejects.toBe(retryFailure); expect(retryFailure.cause).toBeUndefined(); - expect(client.sendCommand).toHaveBeenCalledTimes(1); + expect(client.sendCommand).toHaveBeenCalledTimes(2); }); it("never writes to the rejection even when one instance rejects both dispatches", async () => { @@ -589,7 +484,7 @@ describe("node-redis adapter", () => { // both dispatches, the adapter writes nothing to it. const client = fakeClient(); const shared = new Error("socket torn down"); - client.dialcacheInvalidate.mockRejectedValueOnce(shared); + client.sendCommand.mockRejectedValueOnce(shared); client.sendCommand.mockRejectedValueOnce(shared); const adapter = createNodeRedisDialCacheClient(client as never); @@ -602,7 +497,7 @@ describe("node-redis adapter", () => { it("passes a non-Error invalidation retry rejection through as-is", async () => { const client = fakeClient(); - client.dialcacheInvalidate.mockRejectedValueOnce(new Error("NOPERM evalsha denied")); + client.sendCommand.mockRejectedValueOnce(new Error("NOPERM evalsha denied")); client.sendCommand.mockRejectedValueOnce("socket closed"); const adapter = createNodeRedisDialCacheClient(client as never); @@ -612,10 +507,8 @@ describe("node-redis adapter", () => { }); it("validates the invalidation retry reply through the shared validator", async () => { - // The retry bypasses the registered transformReply, so the trailing - // validator is the only guard on this path. const client = fakeClient({ eval: 0 }); - client.dialcacheInvalidate.mockRejectedValueOnce(new Error("NOPERM evalsha denied")); + client.sendCommand.mockRejectedValueOnce(new Error("NOPERM evalsha denied")); const adapter = createNodeRedisDialCacheClient(client as never); await expectProtocolError( @@ -625,17 +518,11 @@ describe("node-redis adapter", () => { })), "Invalid DialCache Redis invalidate reply; expected integer 1", ); - expect(client.sendCommand).toHaveBeenCalledTimes(1); + expect(client.sendCommand).toHaveBeenCalledTimes(2); }); it("does not retry an invalidation reply-domain violation", async () => { - // The registered transformReply validates inside the returned promise on - // a real client, so a domain violation arrives as a rejection; it is - // deterministic and must surface without a second dispatch. - const client = fakeClient(); - client.dialcacheInvalidate.mockRejectedValueOnce( - new DialCacheRedisProtocolError("Invalid DialCache Redis invalidate reply; expected integer 1"), - ); + const client = fakeClient({ evalSha: 0 }); const adapter = createNodeRedisDialCacheClient(client as never); await expectProtocolError( @@ -645,25 +532,7 @@ describe("node-redis adapter", () => { })), "Invalid DialCache Redis invalidate reply; expected integer 1", ); - expect(client.sendCommand).not.toHaveBeenCalled(); - }); - - it("validates replies at the public node-redis script transform boundary", () => { - expect(dialcacheRedisScripts.dialcacheWriteTrackedStamp.transformReply(0)).toBe(0); - expect(dialcacheRedisScripts.dialcacheWriteTrackedStamp.transformReply(1)).toBe(1); - expect(dialcacheRedisScripts.dialcacheWriteTrackedStamp.transformReply(2)).toBe(2); - expect(dialcacheRedisScripts.dialcacheInvalidate.transformReply(1)).toBe(1); - - for (const reply of INVALID_WRITE_REPLIES) { - expect(() => dialcacheRedisScripts.dialcacheWriteTrackedStamp.transformReply(reply as number)).toThrow( - DialCacheRedisProtocolError, - ); - } - for (const reply of INVALID_INVALIDATION_REPLIES) { - expect(() => dialcacheRedisScripts.dialcacheInvalidate.transformReply(reply as number)).toThrow( - DialCacheRedisProtocolError, - ); - } + expect(client.sendCommand).toHaveBeenCalledTimes(1); }); it("keeps protocol error instanceof checks specific to the base class and subclasses", () => { @@ -684,37 +553,8 @@ describe("node-redis adapter", () => { expect(falselyBranded).not.toBeInstanceOf(DialCacheRedisProtocolError); }); - it("keeps placeholder-lost errors branded and disjoint from protocol errors", () => { - class SpecializedPlaceholderLostError extends DialCacheRedisPlaceholderLostError {} - - const baseError = new DialCacheRedisPlaceholderLostError("base"); - const specializedError = new SpecializedPlaceholderLostError("specialized"); - const crossBundleError = Object.defineProperty( - new Error("lost"), - Symbol.for("dialcache.DialCacheRedisPlaceholderLostError"), - { value: true }, - ); - const falselyBranded = Object.defineProperty( - {}, - Symbol.for("dialcache.DialCacheRedisPlaceholderLostError"), - { value: false }, - ); - - expect(baseError).toBeInstanceOf(DialCacheRedisPlaceholderLostError); - expect(baseError).not.toBeInstanceOf(SpecializedPlaceholderLostError); - expect(specializedError).toBeInstanceOf(SpecializedPlaceholderLostError); - expect(specializedError).toBeInstanceOf(DialCacheRedisPlaceholderLostError); - expect(crossBundleError).toBeInstanceOf(DialCacheRedisPlaceholderLostError); - expect(falselyBranded).not.toBeInstanceOf(DialCacheRedisPlaceholderLostError); - // The benign race-loser class must stay disjoint from operational - // protocol failures, or filtering one silently swallows the other. - expect(baseError).not.toBeInstanceOf(DialCacheRedisProtocolError); - expect(new DialCacheRedisProtocolError("operational")) - .not.toBeInstanceOf(DialCacheRedisPlaceholderLostError); - }); - it("surfaces protocol failures through the normal DialCache observability path", async () => { - const redisClient = createNodeRedisDialCacheClient(fakeClient({ set: 2, invalidate: 0 }) as never); + const redisClient = createNodeRedisDialCacheClient(fakeClient({ set: 2, evalSha: 0 }) as never); const logger = { debug: vi.fn(), warn: vi.fn(), error: vi.fn() }; const metrics = { request: vi.fn(), diff --git a/test/prometheus.test.ts b/test/prometheus.test.ts index bcfebbc..808b69f 100644 --- a/test/prometheus.test.ts +++ b/test/prometheus.test.ts @@ -43,6 +43,7 @@ const METRIC_ERROR_KINDS: Readonly> = { cache_read: true, cache_read_timeout: true, cache_write: true, + tracked_ttl_clamped: true, serialization_load: true, serialization_dump: true, compression: true, @@ -72,7 +73,6 @@ const SHADOW_VALIDATION_OUTCOMES: Readonly mismatch: true, superseded: true, filled: true, - fill_blocked: true, fill_error: true, redis_error: true, source_error: true, @@ -191,6 +191,7 @@ describe("Prometheus metrics adapter", () => { }, 42, ); + metrics.observeFutureTimestampOffset(labels, 0.007); metrics.compression({ ...labels, outcome: "compressed" }); metrics.observeGet(labels, 0.05); metrics.observeFallback(labels, 0.05); @@ -233,6 +234,11 @@ describe("Prometheus metrics adapter", () => { "in_fallback", ]), histogramSchema("schema_dialcache_fallback_timer", ["cache_namespace", "use_case", "key_type", "layer"], TIMER_BUCKETS), + histogramSchema( + "schema_dialcache_future_timestamp_offset_histogram", + ["cache_namespace", "use_case", "key_type", "layer"], + FUTURE_TIMESTAMP_OFFSET_BUCKETS, + ), histogramSchema("schema_dialcache_get_timer", ["cache_namespace", "use_case", "key_type", "layer"], TIMER_BUCKETS), counterSchema("schema_dialcache_invalidation_counter", ["cache_namespace", "key_type", "layer"]), counterSchema("schema_dialcache_miss_counter", ["cache_namespace", "use_case", "key_type", "layer"]), @@ -271,6 +277,20 @@ describe("Prometheus metrics adapter", () => { outcome: "match", }); + const futureTimestampOffset = families.find( + ({ name }) => name === "schema_dialcache_future_timestamp_offset_histogram", + ); + const futureTimestampOffsetSum = futureTimestampOffset?.values.find( + ({ metricName }) => metricName === "schema_dialcache_future_timestamp_offset_histogram_sum", + ); + expect(futureTimestampOffsetSum?.value).toBe(0.007); + expect(futureTimestampOffsetSum?.labels).toEqual({ + cache_namespace: labels.cacheNamespace, + use_case: labels.useCase, + key_type: labels.keyType, + layer: labels.layer, + }); + const serialization = families.find(({ name }) => name === "schema_dialcache_serialization_timer"); const serializationLabels = serialization?.values .filter(({ metricName }) => metricName === `${serialization.name}_sum`) @@ -293,6 +313,31 @@ describe("Prometheus metrics adapter", () => { ]); }); + it("ignores non-finite and non-positive future timestamp offsets", async () => { + const registry = new Registry(); + const metrics = new PrometheusDialCacheMetrics({ registry, prefix: "finite_skew_" }); + const labels = { + cacheNamespace: "users", + useCase: "PrometheusFiniteFutureOffset", + keyType: "user_id", + layer: CacheLayer.REMOTE, + } as const; + + metrics.observeFutureTimestampOffset(labels, 0.25); + for (const invalid of [Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY, 0, -1]) { + metrics.observeFutureTimestampOffset(labels, invalid); + } + + const families = (await registry.getMetricsAsJSON()) as unknown as MetricFamily[]; + const family = families.find(({ name }) => name === "finite_skew_dialcache_future_timestamp_offset_histogram"); + expect(family?.values.find(({ metricName }) => + metricName === "finite_skew_dialcache_future_timestamp_offset_histogram_count" + )?.value).toBe(1); + expect(family?.values.find(({ metricName }) => + metricName === "finite_skew_dialcache_future_timestamp_offset_histogram_sum" + )?.value).toBe(0.25); + }); + it("exports every bounded error category without rewriting labels", async () => { const registry = new Registry(); const metrics = new PrometheusDialCacheMetrics({ registry, prefix: "error_kind_" }); @@ -670,6 +715,26 @@ describe("Prometheus metrics adapter", () => { }); const TIMER_BUCKETS = [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, "+Inf"]; +const FUTURE_TIMESTAMP_OFFSET_BUCKETS = [ + 0.001, + 0.005, + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1, + 5, + 15, + 60, + 300, + 900, + 3_600, + 10_800, + 43_200, + "+Inf", +]; const SIZE_BUCKETS = [100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000, "+Inf"]; const RATIO_BUCKETS = [0.05, 0.1, 0.2, 0.3, 0.5, 0.7, 0.9, 1, "+Inf"]; const VALUE_AGE_BUCKETS = [1, 5, 15, 60, 300, 900, 3_600, 10_800, 43_200, 86_400, 259_200, 604_800, "+Inf"]; diff --git a/test/redis-cluster.integration.test.ts b/test/redis-cluster.integration.test.ts index bdffb16..40b3c43 100644 --- a/test/redis-cluster.integration.test.ts +++ b/test/redis-cluster.integration.test.ts @@ -7,10 +7,10 @@ import { type StartedTestContainer, Wait, } from "testcontainers"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { CacheLayer, DialCache, DialCacheKeyConfig, type DialCacheRedisClient } from "../src/index.js"; -import { createNodeRedisDialCacheClient, dialcacheRedisScripts } from "../src/node-redis.js"; +import { createNodeRedisDialCacheClient } from "../src/node-redis.js"; import { createValkeyGlideDialCacheClient } from "../src/valkey-glide.js"; const remoteOnly = new DialCacheKeyConfig({ @@ -18,11 +18,7 @@ const remoteOnly = new DialCacheKeyConfig({ ramp: { [CacheLayer.REMOTE]: 100 }, }); -const createTestCluster = (options: RedisClusterOptions) => - createCluster({ - ...options, - scripts: dialcacheRedisScripts, - }); +const createTestCluster = (options: RedisClusterOptions) => createCluster(options); async function waitForCluster(container: StartedTestContainer): Promise { for (let attempt = 0; attempt < 50; attempt += 1) { @@ -137,7 +133,7 @@ describe("DialCache Redis protocol on Redis Cluster", () => { await network?.stop(); }); - it("routes cache operations across slots and reloads mutation scripts per node", async () => { + it("routes cache operations across slots and reloads invalidation scripts per node", async () => { if (cluster === undefined) { throw new Error("Redis Cluster did not start"); } @@ -153,8 +149,6 @@ describe("DialCache Redis protocol on Redis Cluster", () => { keyType: "item_id", useCase: "ClusterSlots", cacheKey: (id) => id, - // Tracked, so the pre-flush pass loads the stamp script on every master - // and the post-flush pass proves a genuine per-node NOSCRIPT reload. trackForInvalidation: true, defaultConfig: remoteOnly, }); @@ -172,18 +166,8 @@ describe("DialCache Redis protocol on Redis Cluster", () => { await client.scriptFlush(); }), ); - const recoveryDialcache = new DialCache({ - namespace: "cluster-cache-recovery", - redis: { client: scriptClient, readTimeoutMs: 10_000 }, - }); - const recoverValue = recoveryDialcache.cached(async (id: string) => ({ id, calls: ++calls }), { - keyType: "item_id", - useCase: "ClusterSlots", - cacheKey: (id) => id, - trackForInvalidation: true, - defaultConfig: remoteOnly, - }); - const second = await recoveryDialcache.enable(async () => await Promise.all(ids.map(recoverValue))); + await Promise.all(ids.map(async (id) => await dialcache.invalidateRemote("item_id", id))); + const second = await dialcache.enable(async () => await Promise.all(ids.map(getValue))); const sizesAfterRecovery = await Promise.all( activeCluster.masters.map(async (master) => { const client = await activeCluster.nodeClient(master); @@ -191,7 +175,7 @@ describe("DialCache Redis protocol on Redis Cluster", () => { }), ); const callsAfterRecovery = calls; - const third = await recoveryDialcache.enable(async () => await Promise.all(ids.map(recoverValue))); + const third = await dialcache.enable(async () => await Promise.all(ids.map(getValue))); expect(first.map(({ id }) => id)).toEqual(ids); expect(sizesBeforeFlush.every((size) => size > 0)).toBe(true); @@ -208,7 +192,6 @@ describe("DialCache Redis protocol on Redis Cluster", () => { if (cluster === undefined) { throw new Error("Redis Cluster did not start"); } - expect(dialcacheRedisScripts.dialcacheWriteTrackedStamp.SHA1).not.toBe(dialcacheRedisScripts.dialcacheInvalidate.SHA1); const scriptClient: DialCacheRedisClient = createNodeRedisDialCacheClient(cluster); const dialcache = new DialCache({ namespace: "cluster-cache", @@ -239,14 +222,6 @@ describe("DialCache Redis protocol on Redis Cluster", () => { watermarkKey: "{slot-b}:watermark", }), ).rejects.toThrow(/CROSSSLOT/); - await expect( - scriptClient.write({ - valueKey: "{slot-a}:value", - watermarkKey: "{slot-b}:watermark", - cacheTtlMs: 60_000, - value: "cross", - }), - ).rejects.toThrow(/CROSSSLOT/); }); it("round-trips binary payloads through cluster routing", async () => { @@ -257,7 +232,7 @@ describe("DialCache Redis protocol on Redis Cluster", () => { const valueKey = "binary-cluster:{item:untracked}:value"; const payload = Buffer.from(Array.from({ length: 256 }, (_, index) => index)); - expect(await scriptClient.write({ valueKey, cacheTtlMs: 60_000, value: payload })).toBe(true); + await expect(scriptClient.write({ valueKey, cacheTtlMs: 60_000, value: payload })).resolves.toBeUndefined(); const untrackedRead = await scriptClient.read({ valueKey }); expect(untrackedRead?.payload).toEqual(payload); expect(untrackedRead?.createdAtMs).toBeGreaterThan(0); @@ -270,17 +245,22 @@ describe("DialCache Redis protocol on Redis Cluster", () => { const trackedValueKey = "binary-cluster:{item:tracked}:value"; const watermarkKey = "binary-cluster:{item:tracked}:watermark"; const trackedPayload = Buffer.from([0, 0xff, 0xc3, 0x28, 0x80]); - expect( - await scriptClient.write({ - valueKey: trackedValueKey, - watermarkKey, - cacheTtlMs: 60_000, - value: trackedPayload, - }), - ).toBe(true); + const trackedCreatedAtMs = 1_700_000_000_123; + const now = vi.spyOn(Date, "now").mockReturnValue(trackedCreatedAtMs); + try { + await expect( + scriptClient.write({ + valueKey: trackedValueKey, + cacheTtlMs: 60_000, + value: trackedPayload, + }), + ).resolves.toBeUndefined(); + } finally { + now.mockRestore(); + } const trackedRead = await scriptClient.read({ valueKey: trackedValueKey, watermarkKey }); expect(trackedRead?.payload).toEqual(trackedPayload); - expect(trackedRead?.createdAtMs).toBeGreaterThan(0); + expect(trackedRead?.createdAtMs).toBe(trackedCreatedAtMs); }); it("runs GLIDE tracked mutations against the real cluster", async (ctx) => { @@ -291,31 +271,36 @@ describe("DialCache Redis protocol on Redis Cluster", () => { const valueKey = "glide-cluster:{item:tracked}:value"; const watermarkKey = "glide-cluster:{item:tracked}:watermark"; - expect( - await adapter.write({ valueKey, watermarkKey, cacheTtlMs: 60_000, value: "glide" }), - ).toBe(true); - expect((await adapter.read({ valueKey, watermarkKey }))?.payload).toBe("glide"); + const createdAtMs = 1_700_000_000_456; + const now = vi.spyOn(Date, "now").mockReturnValue(createdAtMs); + try { + await expect( + adapter.write({ valueKey, cacheTtlMs: 60_000, value: "glide" }), + ).resolves.toBeUndefined(); + } finally { + now.mockRestore(); + } + expect(await adapter.read({ valueKey, watermarkKey })).toMatchObject({ + payload: "glide", + createdAtMs, + }); await adapter.invalidate({ watermarkKey, futureBufferMs: 0 }); - // The follow-up write's stamp is fenced unless server time passes the - // zero-buffer watermark; the read-null below holds at any margin. + // The existing value remains fenced until a later client-stamped frame is + // written past the zero-buffer watermark. await new Promise((resolve) => setTimeout(resolve, 25)); expect(await adapter.read({ valueKey, watermarkKey })).toBeNull(); - expect( - await adapter.write({ valueKey, watermarkKey, cacheTtlMs: 60_000, value: "glide-2" }), - ).toBe(true); + await expect( + adapter.write({ valueKey, cacheTtlMs: 60_000, value: "glide-2" }), + ).resolves.toBeUndefined(); expect((await adapter.read({ valueKey, watermarkKey }))?.payload).toBe("glide-2"); const untrackedKey = "glide-cluster:{item:untracked}:value"; - expect(await adapter.write({ valueKey: untrackedKey, cacheTtlMs: 60_000, value: "plain" })).toBe(true); + await expect( + adapter.write({ valueKey: untrackedKey, cacheTtlMs: 60_000, value: "plain" }), + ).resolves.toBeUndefined(); expect((await adapter.read({ valueKey: untrackedKey }))?.payload).toBe("plain"); - await expect(adapter.write({ - valueKey: "{glide-a}:value", - watermarkKey: "{glide-b}:watermark", - cacheTtlMs: 60_000, - value: "cross", - })).rejects.toThrow(/CROSSSLOT/i); }); it("recovers GLIDE cluster mutations after SCRIPT FLUSH on every master", async (ctx) => { @@ -336,9 +321,9 @@ describe("DialCache Redis protocol on Redis Cluster", () => { const watermarkKey = "glide-flush:{item:tracked}:watermark"; await flushAllMasters(); - expect( - await adapter.write({ valueKey, watermarkKey, cacheTtlMs: 60_000, value: "recovered" }), - ).toBe(true); + await expect( + adapter.write({ valueKey, cacheTtlMs: 60_000, value: "recovered" }), + ).resolves.toBeUndefined(); expect((await adapter.read({ valueKey, watermarkKey }))?.payload).toBe("recovered"); await flushAllMasters(); diff --git a/test/redis-payload.test.ts b/test/redis-payload.test.ts index 3eb3d12..0da4010 100644 --- a/test/redis-payload.test.ts +++ b/test/redis-payload.test.ts @@ -2,7 +2,6 @@ import { decodeRedisFrame, decodeTrackedRedisFrame, encodeRedisFrame, - encodeTrackedRedisPlaceholder, } from "../src/redis-protocol.js"; import { DialCacheRedisPayloadEncodingError, @@ -64,27 +63,43 @@ describe("Redis frame decoding", () => { } }); - it("validates tracked frames against integer and fractional watermarks", () => { + it("validates tracked frames against safe-integer watermarks", () => { const frame = encodeFrame("cached", 0, 1_000); const decoded = { payload: "cached", createdAtMs: 1_000 }; expect(decodeTrackedRedisFrame(frame, Buffer.from("999"))).toEqual(decoded); - expect(decodeTrackedRedisFrame(frame, Buffer.from("999.5"))).toEqual(decoded); expect(decodeTrackedRedisFrame(frame, Buffer.from("1000"))).toBeNull(); - expect(decodeTrackedRedisFrame(frame, Buffer.from("1000.5"))).toBeNull(); + + const latestFrame = encodeFrame("latest", 0, Number.MAX_SAFE_INTEGER); + expect( + decodeTrackedRedisFrame(latestFrame, Buffer.from(String(Number.MAX_SAFE_INTEGER - 1))), + ).toEqual({ payload: "latest", createdAtMs: Number.MAX_SAFE_INTEGER }); + expect( + decodeTrackedRedisFrame(latestFrame, Buffer.from(String(Number.MAX_SAFE_INTEGER))), + ).toBeNull(); }); - it("treats missing, malformed, and non-finite watermarks as misses", () => { + it("treats a missing watermark as the zero baseline", () => { + const frame = encodeFrame("cached", 0, 1_000); + + expect(decodeTrackedRedisFrame(frame, null)).toEqual({ + payload: "cached", + createdAtMs: 1_000, + }); + }); + + it("treats malformed and non-finite watermarks as misses", () => { const frame = encodeFrame("cached", 0, 1_000); for (const watermark of [ - null, Buffer.from(""), Buffer.from("-1"), Buffer.from("1."), Buffer.from(".1"), Buffer.from("1e2"), Buffer.from("1\n"), + Buffer.from("999.5"), + Buffer.from(String(Number.MAX_SAFE_INTEGER + 1)), Buffer.from("9".repeat(400)), ]) { expect(decodeTrackedRedisFrame(frame, watermark)).toBeNull(); @@ -96,7 +111,9 @@ describe("Redis frame decoding", () => { expect(decodeTrackedRedisFrame(null, Buffer.from("0"))).toBeNull(); expect(decodeTrackedRedisFrame(Buffer.alloc(9), Buffer.from("0"))).toBeNull(); - expect(decodeTrackedRedisFrame(malformedPayload, null)).toBeNull(); + expect(() => decodeTrackedRedisFrame(malformedPayload, null)).toThrow( + DialCacheRedisPayloadEncodingError, + ); expect(decodeTrackedRedisFrame(malformedPayload, Buffer.from("1000"))).toBeNull(); expect(() => decodeTrackedRedisFrame(malformedPayload, Buffer.from("999"))).toThrow( DialCacheRedisPayloadEncodingError, @@ -131,39 +148,7 @@ describe("Redis frame decoding", () => { expect(decodeRedisFrame(zeroStamped)).toEqual({ payload: "pending", createdAtMs: 0 }); }); - it("encodes tracked placeholders that no read path serves", () => { - const { frame, nonce } = encodeTrackedRedisPlaceholder("pending"); - - expect(frame[0]).toBe(0); - expect(nonce.byteLength).toBe(8); - expect(frame.subarray(1, 9)).toEqual(nonce); - expect(frame[9]).toBe(0); - expect(frame.subarray(10).toString("utf8")).toBe("pending"); - expect(decodeRedisFrame(frame)).toBeNull(); - expect(decodeTrackedRedisFrame(frame, null)).toBeNull(); - expect(decodeTrackedRedisFrame(frame, Buffer.from("0"))).toBeNull(); - expect(decodeTrackedRedisFrame(frame, Buffer.from("1"))).toBeNull(); - - const binary = encodeTrackedRedisPlaceholder(Buffer.from([0, 0xff])); - expect(binary.frame[9]).toBe(1); - expect(decodeRedisFrame(binary.frame)).toBeNull(); - }); - - it("mints a distinct nonce for every placeholder", () => { - // The stamp promotes only the placeholder carrying its own nonce, so - // nonce uniqueness is what keeps concurrent same-key writes disjoint. - const mints = Array.from({ length: 32 }, () => encodeTrackedRedisPlaceholder("pending")); - const nonces = new Set(mints.map(({ nonce }) => nonce.toString("hex"))); - - expect(nonces.size).toBe(32); - for (const { frame, nonce } of mints) { - expect(frame.subarray(1, 9)).toEqual(nonce); - } - }); - - it("gates serving on the version byte even for hostile placeholder nonces", () => { - // A nonce that would decode as a huge timestamp must never beat the - // watermark: version 0 alone keeps the frame a miss on both paths. + it("gates serving on the version byte even for hostile header bytes", () => { const hostile = encodeFrame("pending", 0, 1, 0); hostile.fill(0xff, 1, 9); diff --git a/test/redis-real.integration.test.ts b/test/redis-real.integration.test.ts index 645f7d7..54690d1 100644 --- a/test/redis-real.integration.test.ts +++ b/test/redis-real.integration.test.ts @@ -1,3 +1,5 @@ +import { createHash } from "node:crypto"; + import * as valkeyGlide from "@valkey/valkey-glide"; import { commandOptions, createClient } from "redis"; import { GenericContainer, type StartedTestContainer, Wait } from "testcontainers"; @@ -13,13 +15,16 @@ import { type Serializer, } from "../src/index.js"; import { MARKER_ESCAPED_RAW, MARKER_ZSTD_UTF8 } from "../src/internal/compression.js"; +import { + MAX_SUPPORTED_DURATION_MS, + MAX_TRACKED_REDIS_VALUE_TTL_MS, +} from "../src/internal/duration.js"; import { markerCollidingSerializer, type Row } from "./marker-colliding-serializer.js"; import { INVALIDATE_CACHE_SCRIPT, - WRITE_TRACKED_STAMP_SCRIPT, + MIN_WATERMARK_TTL_MS, } from "../src/internal/redis-scripts.js"; -import { encodeTrackedRedisPlaceholder } from "../src/redis-protocol.js"; -import { createNodeRedisDialCacheClient, dialcacheRedisScripts } from "../src/node-redis.js"; +import { createNodeRedisDialCacheClient } from "../src/node-redis.js"; import { createValkeyGlideDialCacheClient } from "../src/valkey-glide.js"; const engines = [ @@ -32,8 +37,8 @@ const adapterKinds = [ { kind: "valkeyGlide", name: "Valkey GLIDE" }, ] as const; type AdapterKind = (typeof adapterKinds)[number]["kind"]; -const MAX_SUPPORTED_DURATION_MS = 31_536_000_000; const WATERMARK_TTL_MARGIN_MS = 60_000; +const INVALIDATE_CACHE_SHA1 = createHash("sha1").update(INVALIDATE_CACHE_SCRIPT).digest("hex"); interface Deferred { readonly promise: Promise; @@ -53,18 +58,16 @@ const remoteOnly = new DialCacheKeyConfig({ ramp: { [CacheLayer.REMOTE]: 100 }, }); -const createTestClient = (url: string) => createClient({ url, scripts: dialcacheRedisScripts }); +const createTestClient = (url: string) => createClient({ url }); type NodeRedisTestClient = ReturnType; interface RawRedisScriptClient { - /** Invoke only the tracked stamp script, as if its paired placeholder SET was lost. */ - stamp(valueKey: string, watermarkKey: string, cacheTtlMs: number, nonce: Buffer): Promise; - invalidate(watermarkKey: string, futureBufferMs: number): Promise; + invalidate(watermarkKey: string, futureBufferMs: number, invalidatedAtMs: number): Promise; } interface RedisAdapterHarness { readonly adapter: DialCacheRedisClient; - /** Exercise Lua argument validation and stamp states the semantic adapter cannot represent. */ + /** Exercise invalidation Lua argument validation directly. */ readonly raw: RawRedisScriptClient; dispose(): void; } @@ -73,8 +76,20 @@ function createNodeRedisHarness(client: NodeRedisTestClient): RedisAdapterHarnes return { adapter: createNodeRedisDialCacheClient(client), raw: { - stamp: async (...args) => await client.dialcacheWriteTrackedStamp(...args), - invalidate: async (...args) => await client.dialcacheInvalidate(...args), + invalidate: async (watermarkKey, futureBufferMs, invalidatedAtMs) => { + const reply = await client.sendCommand([ + "EVAL", + INVALIDATE_CACHE_SCRIPT, + "1", + watermarkKey, + String(futureBufferMs), + String(invalidatedAtMs), + ]); + if (typeof reply !== "number") { + throw new Error("Unexpected non-integer reply from DialCache test script"); + } + return reply; + }, }, dispose: () => undefined, }; @@ -82,10 +97,7 @@ function createNodeRedisHarness(client: NodeRedisTestClient): RedisAdapterHarnes function createValkeyGlideHarness(client: valkeyGlide.GlideClient): RedisAdapterHarness { const adapter = createValkeyGlideDialCacheClient(client, valkeyGlide); - const rawScripts = { - stamp: new valkeyGlide.Script(WRITE_TRACKED_STAMP_SCRIPT), - invalidate: new valkeyGlide.Script(INVALIDATE_CACHE_SCRIPT), - }; + const invalidationScript = new valkeyGlide.Script(INVALIDATE_CACHE_SCRIPT); const invoke = async ( script: valkeyGlide.Script, keys: Array, @@ -105,19 +117,15 @@ function createValkeyGlideHarness(client: valkeyGlide.GlideClient): RedisAdapter return { adapter, raw: { - stamp: async (valueKey, watermarkKey, cacheTtlMs, nonce) => - await invoke(rawScripts.stamp, [valueKey, watermarkKey], [String(cacheTtlMs), nonce]), - invalidate: async (watermarkKey, futureBufferMs) => + invalidate: async (watermarkKey, futureBufferMs, invalidatedAtMs) => await invoke( - rawScripts.invalidate, + invalidationScript, [watermarkKey], - [String(futureBufferMs)], + [String(futureBufferMs), String(invalidatedAtMs)], ), }, dispose() { - for (const script of Object.values(rawScripts)) { - script.release(); - } + invalidationScript.release(); }, }; } @@ -227,6 +235,33 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(inlineCalls).toBe(1); }); + it("stores the exact application timestamp in tracked frame v1", async () => { + if (client === undefined || admin === undefined) { + throw new Error("Redis test clients did not start"); + } + const valueKey = "client-clock:{item:exact}:value"; + const watermarkKey = "client-clock:{item:exact}:watermark"; + const createdAtMs = 1_700_000_000_123; + const now = vi.spyOn(Date, "now").mockReturnValue(createdAtMs); + try { + await expect(client.adapter.write({ + valueKey, + cacheTtlMs: 60_000, + value: "tracked", + })).resolves.toBeUndefined(); + } finally { + now.mockRestore(); + } + + const stored = await admin.get(commandOptions({ returnBuffers: true }), valueKey); + expect(stored?.[0]).toBe(1); + expect(stored?.readBigUInt64BE(1)).toBe(BigInt(createdAtMs)); + await expect(client.adapter.read({ valueKey, watermarkKey })).resolves.toEqual({ + payload: "tracked", + createdAtMs, + }); + }); + it("compresses values above the threshold and stores small values byte-identical", async () => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); @@ -277,10 +312,8 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); } - // Compression envelopes and the tracked placeholder protocol were built - // in separate branches; this pins their combination: a zstd payload - // rides an unreadable nonce placeholder, gets promoted by the stamp, - // and stays fenceable by the watermark. + // This pins the combination of compression and tracked reads: the + // complete zstd frame remains fenceable by the watermark. const scriptClient: DialCacheRedisClient = client.adapter; const namespace = "real-compression-tracked"; const dialcache = new DialCache({ namespace, redis: { client: scriptClient, readTimeoutMs: 10_000 } }); @@ -309,14 +342,14 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { await dialcache.invalidateRemote("item_id", "big"); // Leave the zero-buffer watermark clearly in the past so the refill's - // stamp cannot land inside the fence window and blank the entry. + // client timestamp lands after it. await new Promise((resolve) => setTimeout(resolve, 25)); const refreshed = await dialcache.enable(async () => await getLarge("big")); expect(refreshed).toEqual({ ...first, calls: 2 }); // The refill must be a published, servable zstd frame: a third read - // serves it from Redis without reloading, and the stored bytes carry a - // promoted version byte with the envelope intact after the stamp. + // serves it from Redis without reloading, and the stored bytes carry the + // complete frame with its compression envelope intact. const third = await dialcache.enable(async () => await getLarge("big")); expect(third).toEqual(refreshed); expect(calls).toBe(2); @@ -373,7 +406,9 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { for (const [index, payload] of payloads.entries()) { const valueKey = `binary-raw:{item:${index}}:value`; - expect(await scriptClient.write({ valueKey, cacheTtlMs: 60_000, value: payload })).toBe(true); + await expect( + scriptClient.write({ valueKey, cacheTtlMs: 60_000, value: payload }), + ).resolves.toBeUndefined(); const roundTrip = await scriptClient.read({ valueKey }); const stored = await admin.get(commandOptions({ returnBuffers: true }), valueKey); @@ -391,14 +426,13 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { const trackedValueKey = "binary-raw:{item:tracked}:value"; const watermarkKey = "binary-raw:{item:tracked}:watermark"; const trackedPayload = Buffer.from([0, 0xff, 0xc3, 0x28, 0x80]); - expect( - await scriptClient.write({ + await expect( + scriptClient.write({ valueKey: trackedValueKey, - watermarkKey, cacheTtlMs: 60_000, value: trackedPayload, }), - ).toBe(true); + ).resolves.toBeUndefined(); const trackedRead = await scriptClient.read({ valueKey: trackedValueKey, watermarkKey }); expect(trackedRead?.payload).toEqual(trackedPayload); expect(trackedRead?.createdAtMs).toBeGreaterThan(0); @@ -560,7 +594,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { keyType: "item_id", outcome: "superseded", }); - // The stored frame was stamped with createdAtMs=1, so both verdicts see + // The stored frame has createdAtMs=1, so both verdicts see // a huge positive age; superseded outcomes record none. expect(metrics.observeShadowValueAge).toHaveBeenCalledTimes(2); expect(metrics.observeShadowValueAge).toHaveBeenNthCalledWith( @@ -795,7 +829,6 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { valueKey, cacheTtlMs: 60_000, value: JSON.stringify(sourceValue), - ...(tracked ? { watermarkKey } : {}), }); expect((await client.adapter.read({ valueKey, @@ -803,13 +836,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { }))?.payload).toBe(JSON.stringify(sourceValue)); expect(await admin.pTTL(valueKey)).toBeGreaterThan(55_000); expect(await admin.pTTL(valueKey)).toBeLessThanOrEqual(60_000); - if (tracked) { - expect(await admin.get(watermarkKey)).toBe("0"); - expect(await admin.pTTL(watermarkKey)).toBeGreaterThan(115_000); - expect(await admin.pTTL(watermarkKey)).toBeLessThanOrEqual(120_000); - } else { - expect(await admin.exists(watermarkKey)).toBe(0); - } + expect(await admin.exists(watermarkKey)).toBe(0); expect(metrics.shadowValidation).toHaveBeenCalledOnce(); expect(metrics.shadowValidation).toHaveBeenCalledWith({ cacheNamespace: namespace, @@ -852,14 +879,14 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(invalidate).not.toHaveBeenCalled(); }); - it("reports a future-watermark-blocked shadow fill without populating Redis", async () => { + it("stores but fences a shadow fill behind a future watermark", async () => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); } - const namespace = "real-dark-shadow-blocked"; - const useCase = "RealDarkShadowBlocked"; - const valueKey = `{${namespace}:item_id:blocked}#${useCase}:dialcache-frame-v1`; - const watermarkKey = `{${namespace}:item_id:blocked}#watermark`; + const namespace = "real-dark-shadow-fenced"; + const useCase = "RealDarkShadowFenced"; + const valueKey = `{${namespace}:item_id:fenced}#${useCase}:dialcache-frame-v1`; + const watermarkKey = `{${namespace}:item_id:fenced}#watermark`; await client.adapter.invalidate({ watermarkKey, futureBufferMs: 60_000 }); const watermarkBefore = await admin.get(watermarkKey); @@ -867,7 +894,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { const write = vi.fn(client.adapter.write); const invalidate = vi.fn(client.adapter.invalidate); const redisClient: DialCacheRedisClient = { ...client.adapter, read, write, invalidate }; - const fillBlocked = deferred(); + const filled = deferred(); const metrics: DialCacheMetricsAdapter = { request: vi.fn(), miss: vi.fn(), @@ -876,8 +903,8 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { invalidation: vi.fn(), coalesced: vi.fn(), shadowValidation: vi.fn(({ outcome }) => { - if (outcome === "fill_blocked") { - fillBlocked.resolve(); + if (outcome === "filled") { + filled.resolve(); } }), observeGet: vi.fn(), @@ -890,12 +917,12 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { redis: { client: redisClient, readTimeoutMs: 10_000 }, metrics, }); - const sourceValue = { id: "blocked", version: 1 }; + const sourceValue = { id: "fenced", version: 1 }; const source = vi.fn(async () => sourceValue); const getPayload = dialcache.cached(source, { keyType: "item_id", useCase, - cacheKey: () => "blocked", + cacheKey: () => "fenced", trackForInvalidation: true, defaultConfig: new DialCacheKeyConfig({ ttlSec: { [CacheLayer.REMOTE]: 60 }, @@ -906,24 +933,25 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { const result = await dialcache.enable(async () => await getPayload()); expect(result).toBe(sourceValue); - await fillBlocked.promise; + await filled.promise; expect(source).toHaveBeenCalledOnce(); expect(read).toHaveBeenCalledOnce(); expect(write).toHaveBeenCalledOnce(); expect(invalidate).not.toHaveBeenCalled(); - expect(await admin.exists(valueKey)).toBe(0); + expect(await admin.exists(valueKey)).toBe(1); + expect(await client.adapter.read({ valueKey, watermarkKey })).toBeNull(); expect(await admin.get(watermarkKey)).toBe(watermarkBefore); expect(metrics.shadowValidation).toHaveBeenCalledOnce(); expect(metrics.shadowValidation).toHaveBeenCalledWith({ cacheNamespace: namespace, useCase, keyType: "item_id", - outcome: "fill_blocked", + outcome: "filled", }); }); - it("reloads every mutation script after SCRIPT FLUSH", async () => { + it("keeps native writes working and reloads invalidation after SCRIPT FLUSH", async () => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); } @@ -931,28 +959,22 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { const valueKey = "script-recovery:{item:untracked}:value"; await admin.scriptFlush(); - expect(await scriptClient.write({ valueKey, cacheTtlMs: 60_000, value: "untracked" })).toBe(true); + await expect( + scriptClient.write({ valueKey, cacheTtlMs: 60_000, value: "untracked" }), + ).resolves.toBeUndefined(); expect((await scriptClient.read({ valueKey }))?.payload).toBe("untracked"); const trackedValueKey = "script-recovery:{item:tracked}:value"; const watermarkKey = "script-recovery:{item:tracked}:watermark"; await admin.scriptFlush(); - expect( - await scriptClient.write({ + await expect( + scriptClient.write({ valueKey: trackedValueKey, - watermarkKey, cacheTtlMs: 60_000, value: "tracked", }), - ).toBe(true); + ).resolves.toBeUndefined(); expect((await scriptClient.read({ valueKey: trackedValueKey, watermarkKey }))?.payload).toBe("tracked"); - // The recovered write must cache the stamp under sha1(source) — the - // digest node-redis registers and the GLIDE batch dispatches — so later - // writes take the single-round-trip path. (The unit suites pin each - // adapter's dispatched digest to an independently computed sha1.) - expect( - await admin.scriptExists(dialcacheRedisScripts.dialcacheWriteTrackedStamp.SHA1), - ).toEqual([true]); await admin.scriptFlush(); await expect( scriptClient.invalidate({ @@ -960,10 +982,11 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { futureBufferMs: 0, }), ).resolves.toBeUndefined(); + expect(await admin.scriptExists(INVALIDATE_CACHE_SHA1)).toEqual([true]); expect(await scriptClient.read({ valueKey: trackedValueKey, watermarkKey })).toBeNull(); }); - it("treats every invalid read frame and watermark state as a miss", async () => { + it("uses zero for a missing watermark and misses on malformed or fenced state", async () => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); } @@ -980,7 +1003,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(await scriptClient.read({ valueKey })).toBeNull(); await admin.set(valueKey, encodeFrame("tracked", 0, 1_000)); - expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); + expect((await scriptClient.read({ valueKey, watermarkKey }))?.payload).toBe("tracked"); await admin.set(watermarkKey, "not-a-watermark"); expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); @@ -992,7 +1015,10 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); await admin.set(watermarkKey, "999.5"); - expect((await scriptClient.read({ valueKey, watermarkKey }))?.payload).toBe("tracked"); + expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); + + await admin.set(watermarkKey, String(Number.MAX_SAFE_INTEGER + 1)); + expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); }); it("records a stale tracked frame as a remote miss without a read error", async () => { @@ -1040,7 +1066,6 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { keyType: "item_id", layer: CacheLayer.REMOTE, } as const; - await expect(dialcache.enable(async () => await getValue())).resolves.toEqual({ source: "fallback" }); expect(fallback).toHaveBeenCalledOnce(); @@ -1055,7 +1080,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(metrics.error).not.toHaveBeenCalled(); }); - it("uses native wrong-type semantics and repairs tracked value keys", async () => { + it("uses native wrong-type read semantics and repairs wrong-type keys", async () => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); } @@ -1071,7 +1096,25 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { await admin.del([valueKey, watermarkKey]); await admin.set(valueKey, encodeFrame("cached", 0, 1_000)); await admin.hSet(watermarkKey, "field", "value"); - await expect(scriptClient.read({ valueKey, watermarkKey })).resolves.toBeNull(); + await expect(scriptClient.read({ valueKey, watermarkKey })).resolves.toEqual({ + payload: "cached", + createdAtMs: 1_000, + }); + const invalidatedAtMs = 1_700_000_000_000; + const now = vi.spyOn(Date, "now").mockReturnValue(invalidatedAtMs); + try { + await expect(scriptClient.invalidate({ + watermarkKey, + futureBufferMs: 100, + })).resolves.toBeUndefined(); + } finally { + now.mockRestore(); + } + expect(await admin.type(watermarkKey)).toBe("string"); + expect(await admin.get(watermarkKey)).toBe(String(invalidatedAtMs + 100)); + expect(await admin.pTTL(watermarkKey)).toBeGreaterThan(MIN_WATERMARK_TTL_MS - 1_000); + expect(await admin.pTTL(watermarkKey)).toBeLessThanOrEqual(MIN_WATERMARK_TTL_MS); + expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); const namespace = "wrong-type-repair"; const repairValueKey = `{${namespace}:item_id:repair}#WrongTypeRepair:dialcache-frame-v1`; @@ -1101,7 +1144,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(await admin.type(repairValueKey)).toBe("string"); }); - it("fails open repeatedly when a tracked watermark has the wrong Redis type", async () => { + it("treats a wrong-type tracked watermark as absent without coupling writes", async () => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); } @@ -1110,8 +1153,6 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { const id = "broken"; const valueKey = `{${namespace}:item_id:${id}}#${useCase}:dialcache-frame-v1`; const watermarkKey = `{${namespace}:item_id:${id}}#watermark`; - const frame = encodeFrame("cached", 0, 1_000); - await admin.set(valueKey, frame, { PX: 60_000 }); await admin.hSet(watermarkKey, "field", "value"); const metrics = { @@ -1140,43 +1181,32 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { trackForInvalidation: true, defaultConfig: remoteOnly, }); - const labels = { - cacheNamespace: namespace, - useCase, - keyType: "item_id", - layer: CacheLayer.REMOTE, - } as const; - await expect(dialcache.enable(async () => await getValue())).resolves.toEqual({ source: "fallback", calls: 1, }); await expect(dialcache.enable(async () => await getValue())).resolves.toEqual({ source: "fallback", - calls: 2, + calls: 1, }); - expect(sourceCalls).toBe(2); + expect(sourceCalls).toBe(1); expect(metrics.request).toHaveBeenCalledTimes(2); - expect(metrics.miss).toHaveBeenCalledTimes(2); - expect(metrics.error).toHaveBeenCalledTimes(2); - expect(metrics.error).toHaveBeenNthCalledWith(1, { - ...labels, - error: "cache_write", - inFallback: false, - }); - expect(metrics.error).toHaveBeenNthCalledWith(2, { - ...labels, - error: "cache_write", - inFallback: false, - }); - expect(metrics.error).not.toHaveBeenCalledWith(expect.objectContaining({ error: "cache_read" })); + expect(metrics.miss).toHaveBeenCalledOnce(); + expect(metrics.error).not.toHaveBeenCalled(); expect(await admin.type(watermarkKey)).toBe("hash"); - // The paired SET lands before the stamp fails on the wrong-type watermark, - // so the original frame is replaced by an unreadable version-0 placeholder. + // Writes do not touch the wrong-type watermark and still replace the + // value with a complete frame. Native MGET represents the watermark as + // nil, so the next tracked read applies the same zero baseline as an + // absent watermark. const stored = await admin.get(commandOptions({ returnBuffers: true }), valueKey); - expect(stored?.[0]).toBe(0); - await expect(client.adapter.read({ valueKey, watermarkKey })).resolves.toBeNull(); + expect(stored?.[0]).toBe(1); + expect((await client.adapter.read({ valueKey }))?.payload).toBe( + JSON.stringify({ source: "fallback", calls: 1 }), + ); + expect((await client.adapter.read({ valueKey, watermarkKey }))?.payload).toBe( + JSON.stringify({ source: "fallback", calls: 1 }), + ); }); it("rejects invalid raw script arguments before mutating Redis", async () => { @@ -1186,53 +1216,57 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { const valueKey = "invalid-args:{item:invalid}:value"; const watermarkKey = "invalid-args:{item:invalid}:watermark"; const notANumber = "not-a-number" as unknown as number; + const validTimestampMs = 1_700_000_000_000; - const nonce = Buffer.alloc(8, 1); - await expect(client.raw.stamp(valueKey, watermarkKey, 0, nonce)).rejects.toThrow("invalid DialCache TTL"); - await expect(client.raw.stamp(valueKey, watermarkKey, notANumber, nonce)).rejects.toThrow("invalid DialCache TTL"); - await expect(client.raw.stamp(valueKey, watermarkKey, Number.NaN, nonce)).rejects.toThrow("invalid DialCache TTL"); - await expect(client.raw.stamp(valueKey, watermarkKey, Number.POSITIVE_INFINITY, nonce)).rejects.toThrow( - "invalid DialCache TTL", - ); - await expect(client.raw.stamp(valueKey, watermarkKey, Number.NEGATIVE_INFINITY, nonce)).rejects.toThrow( - "invalid DialCache TTL", - ); - await expect( - client.raw.stamp(valueKey, watermarkKey, MAX_SUPPORTED_DURATION_MS + 1, nonce), - ).rejects.toThrow("invalid DialCache TTL"); - await expect( - client.raw.stamp(valueKey, watermarkKey, Number.MAX_SAFE_INTEGER, nonce), - ).rejects.toThrow("invalid DialCache TTL"); - await expect( - client.raw.stamp(valueKey, watermarkKey, 1_000, Buffer.alloc(7, 1)), - ).rejects.toThrow("invalid DialCache stamp nonce"); - await expect( - client.raw.stamp(valueKey, watermarkKey, 1_000, Buffer.alloc(9, 1)), - ).rejects.toThrow("invalid DialCache stamp nonce"); - // The adapters enforce the same TTL domain before issuing any command. + // The adapters validate native SET TTLs before issuing any command. for (const badTtl of [0, notANumber, Number.NaN, Number.POSITIVE_INFINITY, MAX_SUPPORTED_DURATION_MS + 1]) { await expect( client.adapter.write({ valueKey, cacheTtlMs: badTtl, value: "value" }), ).rejects.toThrow(RangeError); - await expect( - client.adapter.write({ valueKey, watermarkKey, cacheTtlMs: badTtl, value: "value" }), - ).rejects.toThrow(RangeError); } - await expect(client.raw.invalidate(watermarkKey, -1)).rejects.toThrow("invalid DialCache future buffer"); - await expect(client.raw.invalidate(watermarkKey, notANumber)).rejects.toThrow("invalid DialCache future buffer"); - await expect(client.raw.invalidate(watermarkKey, Number.NaN)).rejects.toThrow("invalid DialCache future buffer"); - await expect(client.raw.invalidate(watermarkKey, Number.POSITIVE_INFINITY)).rejects.toThrow( + await expect(client.raw.invalidate(watermarkKey, -1, validTimestampMs)).rejects.toThrow( + "invalid DialCache future buffer", + ); + await expect(client.raw.invalidate(watermarkKey, 1.5, validTimestampMs)).rejects.toThrow( + "invalid DialCache future buffer", + ); + await expect(client.raw.invalidate(watermarkKey, notANumber, validTimestampMs)).rejects.toThrow( + "invalid DialCache future buffer", + ); + await expect(client.raw.invalidate(watermarkKey, Number.NaN, validTimestampMs)).rejects.toThrow( + "invalid DialCache future buffer", + ); + await expect( + client.raw.invalidate(watermarkKey, Number.POSITIVE_INFINITY, validTimestampMs), + ).rejects.toThrow( "invalid DialCache future buffer", ); - await expect(client.raw.invalidate(watermarkKey, Number.NEGATIVE_INFINITY)).rejects.toThrow( + await expect( + client.raw.invalidate(watermarkKey, Number.NEGATIVE_INFINITY, validTimestampMs), + ).rejects.toThrow( "invalid DialCache future buffer", ); await expect( - client.raw.invalidate(watermarkKey, MAX_SUPPORTED_DURATION_MS + 1), + client.raw.invalidate(watermarkKey, MAX_SUPPORTED_DURATION_MS + 1, validTimestampMs), ).rejects.toThrow("invalid DialCache future buffer"); await expect( - client.raw.invalidate(watermarkKey, Number.MAX_SAFE_INTEGER), + client.raw.invalidate(watermarkKey, Number.MAX_SAFE_INTEGER, validTimestampMs), ).rejects.toThrow("invalid DialCache future buffer"); + for (const invalidTimestampMs of [ + -1, + 1.5, + notANumber, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.MAX_SAFE_INTEGER + 1, + ]) { + await expect( + client.raw.invalidate(watermarkKey, 0, invalidTimestampMs), + ).rejects.toThrow("invalid DialCache invalidatedAtMs"); + } + await expect( + client.raw.invalidate(watermarkKey, 1, Number.MAX_SAFE_INTEGER), + ).rejects.toThrow("invalid DialCache invalidatedAtMs"); expect(await admin.exists([valueKey, watermarkKey])).toBe(0); }); @@ -1242,9 +1276,9 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { throw new Error("Redis test clients did not start"); } const valueKey = "maximum-args:{item:untracked}:value"; - expect( - await client.adapter.write({ valueKey, cacheTtlMs: MAX_SUPPORTED_DURATION_MS, value: "value" }), - ).toBe(true); + await expect( + client.adapter.write({ valueKey, cacheTtlMs: MAX_SUPPORTED_DURATION_MS, value: "value" }), + ).resolves.toBeUndefined(); expect(await admin.pTTL(valueKey)).toBeGreaterThan( MAX_SUPPORTED_DURATION_MS - 1_000, ); @@ -1252,71 +1286,42 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { MAX_SUPPORTED_DURATION_MS, ); - const trackedValueKey = "maximum-args:{item:tracked}:value"; - const trackedWatermarkKey = "maximum-args:{item:tracked}:watermark"; - expect( - await client.adapter.write({ - valueKey: trackedValueKey, - watermarkKey: trackedWatermarkKey, - cacheTtlMs: MAX_SUPPORTED_DURATION_MS, - value: "value", - }), - ).toBe(true); - expect(await admin.pTTL(trackedWatermarkKey)).toBeGreaterThan( - MAX_SUPPORTED_DURATION_MS + WATERMARK_TTL_MARGIN_MS - 1_000, - ); - expect(await admin.pTTL(trackedWatermarkKey)).toBeLessThanOrEqual( - MAX_SUPPORTED_DURATION_MS + WATERMARK_TTL_MARGIN_MS, - ); - const invalidationKey = "maximum-args:{item:invalidation}:watermark"; - const beforeMs = (await admin.time()).getTime(); + const invalidatedAtMs = 1_700_000_000_000; expect( - await client.raw.invalidate(invalidationKey, MAX_SUPPORTED_DURATION_MS), + await client.raw.invalidate( + invalidationKey, + MAX_SUPPORTED_DURATION_MS, + invalidatedAtMs, + ), ).toBe(1); - expect(Number(await admin.get(invalidationKey))).toBeGreaterThanOrEqual( - beforeMs + MAX_SUPPORTED_DURATION_MS, + expect(Number(await admin.get(invalidationKey))).toBe( + invalidatedAtMs + MAX_SUPPORTED_DURATION_MS, ); expect(await admin.pTTL(invalidationKey)).toBeGreaterThan( - MAX_SUPPORTED_DURATION_MS + WATERMARK_TTL_MARGIN_MS - 1_000, + MAX_SUPPORTED_DURATION_MS + MAX_TRACKED_REDIS_VALUE_TTL_MS + WATERMARK_TTL_MARGIN_MS - 1_000, ); expect(await admin.pTTL(invalidationKey)).toBeLessThanOrEqual( - MAX_SUPPORTED_DURATION_MS + WATERMARK_TTL_MARGIN_MS, + MAX_SUPPORTED_DURATION_MS + MAX_TRACKED_REDIS_VALUE_TTL_MS + WATERMARK_TTL_MARGIN_MS, ); + + const maximumTimestampKey = "maximum-args:{item:timestamp}:watermark"; + expect( + await client.raw.invalidate(maximumTimestampKey, 0, Number.MAX_SAFE_INTEGER), + ).toBe(1); + expect(await admin.get(maximumTimestampKey)).toBe(String(Number.MAX_SAFE_INTEGER)); }); - it("rounds fractional raw protocol durations upward", async () => { + it("rounds fractional native write TTLs upward", async () => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); } const valueKey = "fractional-args:{item:fractional}:value"; - const watermarkKey = "fractional-args:{item:fractional}:watermark"; - - expect(await client.adapter.write({ valueKey, cacheTtlMs: 1_000.1, value: "value" })).toBe(true); + await expect( + client.adapter.write({ valueKey, cacheTtlMs: 1_000.1, value: "value" }), + ).resolves.toBeUndefined(); expect(await admin.pTTL(valueKey)).toBeGreaterThan(900); expect(await admin.pTTL(valueKey)).toBeLessThanOrEqual(1_001); - - const trackedValueKey = "fractional-args:{item:tracked}:value"; - const trackedWatermarkKey = "fractional-args:{item:tracked}:watermark"; - expect( - await client.adapter.write({ - valueKey: trackedValueKey, - watermarkKey: trackedWatermarkKey, - cacheTtlMs: 1_000.1, - value: "value", - }), - ).toBe(true); - expect(await admin.get(trackedWatermarkKey)).toBe("0"); - expect(await admin.pTTL(trackedWatermarkKey)).toBeGreaterThan(60_000); - expect(await admin.pTTL(trackedWatermarkKey)).toBeLessThanOrEqual(61_001); - - const beforeMs = (await admin.time()).getTime(); - expect(await client.raw.invalidate(watermarkKey, 100.1)).toBe(1); - const watermark = Number(await admin.get(watermarkKey)); - expect(Number.isSafeInteger(watermark)).toBe(true); - expect(watermark).toBeGreaterThanOrEqual(beforeMs + 101); - expect(await admin.pTTL(watermarkKey)).toBeGreaterThan(60_000); - expect(await admin.pTTL(watermarkKey)).toBeLessThanOrEqual(60_101); }); it("keeps native reads working after SCRIPT FLUSH", async () => { @@ -1339,9 +1344,8 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { version = 2; const cached = await dialcache.enable(async () => await getUser("123")); await dialcache.invalidateRemote("user_id", "123"); - // The refill's stamp is fenced unless server time passes the - // zero-buffer watermark; the afterScriptFlush read needs that write to - // have been published (calls must stay 2). + // The refill's client timestamp must advance past the zero-buffer + // watermark for the next tracked read to serve it. await new Promise((resolve) => setTimeout(resolve, 25)); const refreshed = await dialcache.enable(async () => await getUser("123")); await admin.scriptFlush(); @@ -1377,7 +1381,7 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(second).toEqual({ id: "bad", calls: 2 }); }); - it("rejects malformed tracked watermark writes and leaves only an unreadable placeholder", async () => { + it("writes complete frames without inspecting malformed watermarks", async () => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); } @@ -1389,15 +1393,13 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { await admin.set(watermarkKey, malformed, { PX: 60_000 }); await expect(scriptClient.write({ valueKey, - watermarkKey, cacheTtlMs: 60_000, value: "replacement", - })).rejects.toThrow("invalid DialCache watermark"); - // The paired SET lands before the stamp validates the watermark, so the - // tracked path serves nothing and the placeholder stays unpromoted. + })).resolves.toBeUndefined(); expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); + expect((await scriptClient.read({ valueKey }))?.payload).toBe("replacement"); const stored = await admin.get(commandOptions({ returnBuffers: true }), valueKey); - expect(stored?.[0]).toBe(0); + expect(stored?.[0]).toBe(1); await admin.del(valueKey); } }); @@ -1456,63 +1458,37 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { ); }); - it("keeps future fractional watermarks alive without shortening longer TTLs", async () => { - if (client === undefined || admin === undefined) { - throw new Error("Redis test clients did not start"); - } - const scriptClient = client.adapter; - const redisNowMs = (await admin.time()).getTime(); - const legacyWatermark = redisNowMs + 30_000.5; - const shortTtlKey = "legacy:{urn:user_id:short}#watermark"; - await admin.set(shortTtlKey, String(legacyWatermark), { PX: 1_000 }); - - await scriptClient.invalidate({ - watermarkKey: shortTtlKey, - futureBufferMs: 1_000, - }); - - expect(Number(await admin.get(shortTtlKey))).toBeGreaterThanOrEqual(Math.ceil(legacyWatermark)); - expect(await admin.pTTL(shortTtlKey)).toBeGreaterThan(89_000); - - const longTtlKey = "legacy:{urn:user_id:long}#watermark"; - await admin.set(longTtlKey, String(legacyWatermark), { PX: 120_000 }); - const ttlBefore = await admin.pTTL(longTtlKey); - - await scriptClient.invalidate({ - watermarkKey: longTtlKey, - futureBufferMs: 1_000, - }); - - expect(Number(await admin.get(longTtlKey))).toBeGreaterThanOrEqual(Math.ceil(legacyWatermark)); - const ttlAfter = await admin.pTTL(longTtlKey); - expect(ttlAfter).toBeGreaterThan(ttlBefore - 1_000); - expect(ttlAfter).toBeLessThanOrEqual(ttlBefore); - }); - it("creates missing and repairs malformed invalidation watermarks", async () => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); } const scriptClient = client.adapter; const missingKey = "invalidate-paths:{item:missing}:watermark"; - const beforeMs = (await admin.time()).getTime(); - - await scriptClient.invalidate({ watermarkKey: missingKey, futureBufferMs: 100 }); - - const created = Number(await admin.get(missingKey)); - expect(Number.isSafeInteger(created)).toBe(true); - expect(created).toBeGreaterThanOrEqual(beforeMs + 100); - expect(await admin.pTTL(missingKey)).toBeGreaterThan(60_000); - - for (const [suffix, malformed] of [ - ["syntax", "not-a-watermark"], - ["overflow", "9".repeat(400)], - ] as const) { - const watermarkKey = `invalidate-paths:{item:${suffix}}:watermark`; - await admin.set(watermarkKey, malformed, { PX: 1_000 }); - await scriptClient.invalidate({ watermarkKey, futureBufferMs: 0 }); - expect(Number.isSafeInteger(Number(await admin.get(watermarkKey)))).toBe(true); - expect(await admin.pTTL(watermarkKey)).toBeGreaterThan(59_000); + const invalidatedAtMs = 1_700_000_000_000; + const now = vi.spyOn(Date, "now").mockReturnValue(invalidatedAtMs); + try { + await scriptClient.invalidate({ watermarkKey: missingKey, futureBufferMs: 100 }); + + const created = Number(await admin.get(missingKey)); + expect(created).toBe(invalidatedAtMs + 100); + expect(await admin.pTTL(missingKey)).toBeGreaterThan(MIN_WATERMARK_TTL_MS - 1_000); + expect(await admin.pTTL(missingKey)).toBeLessThanOrEqual(MIN_WATERMARK_TTL_MS); + + for (const [suffix, malformed] of [ + ["syntax", "not-a-watermark"], + ["fractional", "1700000030000.5"], + ["unsafe", String(Number.MAX_SAFE_INTEGER + 1)], + ["overflow", "9".repeat(400)], + ] as const) { + const watermarkKey = `invalidate-paths:{item:${suffix}}:watermark`; + await admin.set(watermarkKey, malformed, { PX: 1_000 }); + await scriptClient.invalidate({ watermarkKey, futureBufferMs: 0 }); + expect(Number(await admin.get(watermarkKey))).toBe(invalidatedAtMs); + expect(await admin.pTTL(watermarkKey)).toBeGreaterThan(MIN_WATERMARK_TTL_MS - 1_000); + expect(await admin.pTTL(watermarkKey)).toBeLessThanOrEqual(MIN_WATERMARK_TTL_MS); + } + } finally { + now.mockRestore(); } }); @@ -1530,68 +1506,36 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { expect(await admin.pTTL(watermarkKey)).toBe(-1); }); - it("preserves a fractional legacy watermark while extending its TTL", async () => { - if (client === undefined || admin === undefined) { - throw new Error("Redis test clients did not start"); - } - const scriptClient = client.adapter; - const valueKey = "legacy-write:{urn:user_id:123}:value"; - const watermarkKey = "legacy-write:{urn:user_id:123}:watermark"; - await admin.set(watermarkKey, "1.75", { PX: 1_000 }); - - const wrote = await scriptClient.write({ - valueKey, - watermarkKey, - cacheTtlMs: 2_000, - value: "cached", - }); - - expect(wrote).toBe(true); - expect(await admin.get(watermarkKey)).toBe("1.75"); - expect(await admin.pTTL(watermarkKey)).toBeGreaterThanOrEqual(61_000); - expect((await scriptClient.read({ valueKey, watermarkKey }))?.payload).toBe("cached"); - }); - - it("does not rewrite sufficient or persistent watermarks on tracked writes", async () => { + it("does not create or rewrite watermarks on writes", async () => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); } const scriptClient = client.adapter; - const sufficientValueKey = "write-sufficient:{item:sufficient}:value"; - const sufficientWatermarkKey = "write-sufficient:{item:sufficient}:watermark"; - await admin.set(sufficientWatermarkKey, "1.75", { PX: 120_000 }); - const sufficientTtlBefore = await admin.pTTL(sufficientWatermarkKey); - - expect( - await scriptClient.write({ - valueKey: sufficientValueKey, - watermarkKey: sufficientWatermarkKey, - cacheTtlMs: 2_000, - value: "cached", - }), - ).toBe(true); - - expect(await admin.get(sufficientWatermarkKey)).toBe("1.75"); - expect(await admin.pTTL(sufficientWatermarkKey)).toBeGreaterThan(sufficientTtlBefore - 1_000); - expect(await admin.pTTL(sufficientWatermarkKey)).toBeLessThanOrEqual(sufficientTtlBefore); + const missingValueKey = "write-missing:{item:missing}:value"; + const missingWatermarkKey = "write-missing:{item:missing}:watermark"; + await scriptClient.write({ valueKey: missingValueKey, cacheTtlMs: 2_000, value: "cached" }); + expect(await admin.exists(missingWatermarkKey)).toBe(0); + expect((await scriptClient.read({ + valueKey: missingValueKey, + watermarkKey: missingWatermarkKey, + }))?.payload).toBe("cached"); const persistentValueKey = "write-persistent:{item:persistent}:value"; const persistentWatermarkKey = "write-persistent:{item:persistent}:watermark"; - await admin.set(persistentWatermarkKey, "2.25"); + await admin.set(persistentWatermarkKey, "2"); - expect( - await scriptClient.write({ + await expect( + scriptClient.write({ valueKey: persistentValueKey, - watermarkKey: persistentWatermarkKey, cacheTtlMs: 2_000, value: "cached", }), - ).toBe(true); - expect(await admin.get(persistentWatermarkKey)).toBe("2.25"); + ).resolves.toBeUndefined(); + expect(await admin.get(persistentWatermarkKey)).toBe("2"); expect(await admin.pTTL(persistentWatermarkKey)).toBe(-1); }); - it("atomically blocks writes during the buffer and extends watermark TTL", async () => { + it("fences complete writes during the buffer without extending the watermark", async () => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); } @@ -1600,117 +1544,94 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { const watermarkKey = "protocol:{item:ttl}:watermark"; const writeRequest = { valueKey, - watermarkKey, cacheTtlMs: 2_000, value: "cached", }; + const invalidatedAtMs = 1_700_000_000_000; + const now = vi.spyOn(Date, "now").mockReturnValue(invalidatedAtMs); + try { + await scriptClient.write(writeRequest); + expect(await admin.exists(watermarkKey)).toBe(0); + expect((await scriptClient.read({ valueKey, watermarkKey }))?.payload).toBe("cached"); - expect(await scriptClient.write(writeRequest)).toBe(true); - expect(await admin.get(watermarkKey)).toBe("0"); - const ttlAfterWrite = await admin.pTTL(watermarkKey); - expect(ttlAfterWrite).toBeGreaterThanOrEqual(61_000); - - await scriptClient.invalidate({ watermarkKey, futureBufferMs: 100 }); - expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); - const watermarkBeforeBlockedWrite = await admin.get(watermarkKey); - const watermarkTtlBeforeBlockedWrite = await admin.pTTL(watermarkKey); - expect(await scriptClient.write({ ...writeRequest, value: "blocked" })).toBe(false); - expect(await scriptClient.read({ valueKey })).toBeNull(); - expect(await admin.get(watermarkKey)).toBe(watermarkBeforeBlockedWrite); - const watermarkTtlAfterBlockedWrite = await admin.pTTL(watermarkKey); - expect(watermarkTtlAfterBlockedWrite).toBeGreaterThan(watermarkTtlBeforeBlockedWrite - 1_000); - expect(watermarkTtlAfterBlockedWrite).toBeLessThanOrEqual(watermarkTtlBeforeBlockedWrite); - const ttlBeforeRead = await admin.pTTL(watermarkKey); - await scriptClient.read({ valueKey, watermarkKey }); - expect(await admin.pTTL(watermarkKey)).toBeLessThanOrEqual(ttlBeforeRead); - - await new Promise((resolve) => setTimeout(resolve, 110)); - expect(await scriptClient.write({ ...writeRequest, value: "fresh" })).toBe(true); - expect((await scriptClient.read({ valueKey, watermarkKey }))?.payload).toBe("fresh"); + await scriptClient.invalidate({ watermarkKey, futureBufferMs: 100 }); + expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); + const watermarkBeforeWrite = await admin.get(watermarkKey); + const watermarkTtlBeforeWrite = await admin.pTTL(watermarkKey); + await scriptClient.write({ ...writeRequest, value: "behind-watermark" }); + expect((await scriptClient.read({ valueKey }))?.payload).toBe("behind-watermark"); + expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); + expect(await admin.get(watermarkKey)).toBe(watermarkBeforeWrite); + const watermarkTtlAfterWrite = await admin.pTTL(watermarkKey); + expect(watermarkTtlAfterWrite).toBeGreaterThan(watermarkTtlBeforeWrite - 1_000); + expect(watermarkTtlAfterWrite).toBeLessThanOrEqual(watermarkTtlBeforeWrite); + const ttlBeforeRead = watermarkTtlAfterWrite; + await scriptClient.read({ valueKey, watermarkKey }); + expect(await admin.pTTL(watermarkKey)).toBeLessThanOrEqual(ttlBeforeRead); + + now.mockReturnValue(invalidatedAtMs + 101); + await scriptClient.write({ ...writeRequest, value: "fresh" }); + expect((await scriptClient.read({ valueKey, watermarkKey }))?.payload).toBe("fresh"); + } finally { + now.mockRestore(); + } }); - it("documents that losing a watermark removes its publication fence", async () => { + it("documents that losing a watermark removes its read-time invalidation fence", async () => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); } const scriptClient = client.adapter; const valueKey = "watermark-loss:{item:tracked}:value"; const watermarkKey = "watermark-loss:{item:tracked}:watermark"; - const staleWrite = { - valueKey, - watermarkKey, - cacheTtlMs: 60_000, - value: "stale", - }; + await scriptClient.write({ valueKey, cacheTtlMs: 60_000, value: "stale" }); await scriptClient.invalidate({ watermarkKey, futureBufferMs: 60_000 }); - expect(await scriptClient.write(staleWrite)).toBe(false); + expect(await scriptClient.read({ valueKey, watermarkKey })).toBeNull(); await admin.del(watermarkKey); - expect(await scriptClient.write(staleWrite)).toBe(true); - expect(await admin.get(watermarkKey)).toBe("0"); + expect(await admin.exists(watermarkKey)).toBe(0); expect((await scriptClient.read({ valueKey, watermarkKey }))?.payload).toBe("stale"); }); - it("never serves an unstamped placeholder and refuses foreign stamps", async () => { - if (client === undefined || admin === undefined) { - throw new Error("Redis test clients did not start"); - } - const valueKey = "placeholder:{item:pending}:value"; - const watermarkKey = "placeholder:{item:pending}:watermark"; - const { frame, nonce } = encodeTrackedRedisPlaceholder("pending"); - await admin.set(valueKey, frame, { PX: 60_000 }); - await admin.set(watermarkKey, "0", { PX: 120_000 }); - - expect(await client.adapter.read({ valueKey, watermarkKey })).toBeNull(); - expect(await client.adapter.read({ valueKey })).toBeNull(); - - // A stamp carrying a different write's nonce must not promote this - // placeholder: a leftover from a failed write stays unreadable even - // after later invalidations pass. - expect(await client.raw.stamp(valueKey, watermarkKey, 2_000, Buffer.alloc(8, 0xab))).toBe(2); - expect(await client.adapter.read({ valueKey, watermarkKey })).toBeNull(); - - // Only the paired nonce promotes it to a served, server-stamped frame. - expect(await client.raw.stamp(valueKey, watermarkKey, 2_000, nonce)).toBe(1); - expect((await client.adapter.read({ valueKey, watermarkKey }))?.payload).toBe("pending"); - const stored = await admin.get(commandOptions({ returnBuffers: true }), valueKey); - expect(stored?.[0]).toBe(1); - expect(stored?.readBigUInt64BE(1) ?? 0n).toBeGreaterThan(0n); - }); - - it("refuses to restamp an existing frame after its paired SET was lost", async () => { - if (client === undefined || admin === undefined) { - throw new Error("Redis test clients did not start"); - } - const valueKey = "restamp:{item:fenced}:value"; - const watermarkKey = "restamp:{item:fenced}:watermark"; - // A stale frame fenced by a past invalidation, as left behind when a - // fallback write's SET fails (for example on OOM) but its stamp still runs. - await admin.set(valueKey, encodeFrame("stale", 0, 1_000), { PX: 60_000 }); - await admin.set(watermarkKey, "2000", { PX: 120_000 }); - - expect(await client.raw.stamp(valueKey, watermarkKey, 2_000, Buffer.alloc(8, 1))).toBe(2); - - const stored = await admin.get(commandOptions({ returnBuffers: true }), valueKey); - expect(stored?.readBigUInt64BE(1)).toBe(1_000n); - expect(await client.adapter.read({ valueKey, watermarkKey })).toBeNull(); - }); - - it("does not create a value key when stamping after a lost SET", async () => { - if (client === undefined || admin === undefined) { - throw new Error("Redis test clients did not start"); - } - const valueKey = "stamp-missing:{item:lost}:value"; - const watermarkKey = "stamp-missing:{item:lost}:watermark"; - - expect(await client.raw.stamp(valueKey, watermarkKey, 2_000, Buffer.alloc(8, 2))).toBe(2); + }); - expect(await admin.exists(valueKey)).toBe(0); - expect(await admin.get(watermarkKey)).toBe("0"); - expect(await admin.pTTL(watermarkKey)).toBeGreaterThan(60_000); - }); + it("preserves a watermark when GET fails for a reason other than WRONGTYPE", async () => { + if (admin === undefined) { + throw new Error("Redis test clients did not start"); + } + const username = "dialcache-invalidation-no-get"; + const password = "dialcache-invalidation-test-password"; + const watermarkKey = "invalidation-acl:{item:protected}:watermark"; + const existingWatermark = "1800000000000"; + await admin.set(watermarkKey, existingWatermark, { PX: 60_000 }); + await admin.sendCommand([ + "ACL", + "SETUSER", + username, + "reset", + "on", + `>${password}`, + "~*", + "+eval", + "+set", + "+pttl", + "-get", + ]); + const restricted = admin.duplicate({ username, password }); + restricted.on("error", () => undefined); + try { + await restricted.connect(); + await expect(restricted.eval(INVALIDATE_CACHE_SCRIPT, { + keys: [watermarkKey], + arguments: ["0", "1700000000000"], + })).rejects.toThrow(/ACL|can't run this command|no permissions/); + expect(await admin.get(watermarkKey)).toBe(existingWatermark); + } finally { + await restricted.quit().catch(() => undefined); + await admin.sendCommand(["ACL", "DELUSER", username]); + } }); it("uses one wire format across node-redis and Valkey GLIDE", async () => { @@ -1732,7 +1653,6 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { const nodeTrackedWatermarkKey = "interop:{node-tracked}:watermark"; await nodeRedis.write({ valueKey: nodeTrackedValueKey, - watermarkKey: nodeTrackedWatermarkKey, cacheTtlMs: 60_000, value: binary, }); @@ -1745,7 +1665,6 @@ describe.each(engines)("DialCache Redis protocol on $name", ({ image }) => { const glideTrackedWatermarkKey = "interop:{glide-tracked}:watermark"; await valkeyGlide.write({ valueKey: glideTrackedValueKey, - watermarkKey: glideTrackedWatermarkKey, cacheTtlMs: 60_000, value: "tracked", }); diff --git a/test/valkey-glide.test.ts b/test/valkey-glide.test.ts index cb92929..8fed18f 100644 --- a/test/valkey-glide.test.ts +++ b/test/valkey-glide.test.ts @@ -1,17 +1,18 @@ import { createHash } from "node:crypto"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { DialCacheRedisPayloadEncodingError, DialCacheRedisPayloadError, - DialCacheRedisPlaceholderLostError, DialCacheRedisProtocolError, } from "../src/redis-client.js"; -import { INVALIDATE_CACHE_SCRIPT, WRITE_TRACKED_STAMP_SCRIPT } from "../src/redis-protocol.js"; +import { INVALIDATE_CACHE_SCRIPT } from "../src/redis-protocol.js"; import { createValkeyGlideDialCacheClient } from "../src/valkey-glide.js"; -const INVALID_WRITE_REPLIES: readonly unknown[] = [ +const INVALID_INVALIDATION_REPLIES: readonly unknown[] = [ + 0, + 2, -1, 3, 0.5, @@ -24,24 +25,17 @@ const INVALID_WRITE_REPLIES: readonly unknown[] = [ null, undefined, ]; -const INVALID_INVALIDATION_REPLIES: readonly unknown[] = [0, 2, ...INVALID_WRITE_REPLIES]; const decoderBytes = Symbol("bytes"); const batchInstances: MockBatch[] = []; -const clusterBatchInstances: MockClusterBatch[] = []; const standaloneClients = new WeakSet(); const clusterClients = new WeakSet(); class MockBatch { - readonly commands: Array> = []; readonly mget = vi.fn((keys: Array) => { this.keys = keys; return this; }); - readonly customCommand = vi.fn((args: Array) => { - this.commands.push(args); - return this; - }); keys: Array | undefined; constructor(readonly isAtomic: boolean) { @@ -49,13 +43,6 @@ class MockBatch { } } -class MockClusterBatch extends MockBatch { - constructor(isAtomic: boolean) { - super(isAtomic); - clusterBatchInstances.push(this); - } -} - function mockClientIdentity(instances: WeakSet) { return { [Symbol.hasInstance](value: unknown): boolean { @@ -69,7 +56,6 @@ function mockClientIdentity(instances: WeakSet) { const mockGlide = { Batch: MockBatch, - ClusterBatch: MockClusterBatch, Decoder: { Bytes: decoderBytes }, GlideClient: mockClientIdentity(standaloneClients), GlideClusterClient: mockClientIdentity(clusterClients), @@ -142,7 +128,10 @@ async function expectProtocolError(operation: Promise, message: string) describe("Valkey GLIDE adapter", () => { beforeEach(() => { batchInstances.length = 0; - clusterBatchInstances.length = 0; + }); + + afterEach(() => { + vi.restoreAllMocks(); }); it("uses GET and a non-atomic primary MGET batch that preserves caller WATCH state", async () => { @@ -243,7 +232,7 @@ describe("Valkey GLIDE adapter", () => { ); }); - it("rejects an ambiguous client identity before allocating scripts", () => { + it("rejects an ambiguous client identity", () => { const client = fakeClient(); clusterClients.add(client); @@ -254,24 +243,18 @@ describe("Valkey GLIDE adapter", () => { ); }); - it("requires GLIDE 2.x Batch support before allocating scripts", () => { + it("requires GLIDE 2.x Batch support", () => { const client = fakeClient(); const glideWithoutBatch = { ...mockGlide, Batch: undefined, } as unknown as typeof mockGlide; - const glideWithoutClusterBatch = { - ...mockGlide, - ClusterBatch: undefined, - } as unknown as typeof mockGlide; - for (const runtime of [glideWithoutBatch, glideWithoutClusterBatch]) { - expect( - () => createValkeyGlideDialCacheClient(client, runtime), - ).toThrow( - "Valkey GLIDE DialCache requires @valkey/valkey-glide >=2.0.0 with Batch and ClusterBatch constructors", - ); - } + expect( + () => createValkeyGlideDialCacheClient(client, glideWithoutBatch), + ).toThrow( + "Valkey GLIDE DialCache requires @valkey/valkey-glide >=2.0.0 with a Batch constructor", + ); }); it("preserves GLIDE invocation options when given a core read context", async () => { @@ -290,28 +273,25 @@ describe("Valkey GLIDE adapter", () => { ); }); - it("writes untracked SETs directly and tracked pairs through a batch", async () => { + it("writes string and binary values as complete frames with one SET each", async () => { + const now = vi.spyOn(Date, "now") + .mockReturnValueOnce(1_234) + .mockReturnValueOnce(2_345) + .mockReturnValueOnce(3_456); const binary = Buffer.from([0, 0xff, 0x80]); - const client = fakeClient( - Buffer.from("OK"), - [Buffer.from("OK"), 0], - 1, - ); + const client = fakeClient(Buffer.from("OK"), "OK", 1); const adapter = createValkeyGlideDialCacheClient(client, mockGlide); - const before = Date.now(); await expect( adapter.write({ valueKey: "plain:value", cacheTtlMs: 1_000, value: "hello" }), - ).resolves.toBe(true); - const after = Date.now(); + ).resolves.toBeUndefined(); await expect( adapter.write({ valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", cacheTtlMs: 2_000, value: binary, }), - ).resolves.toBe(false); + ).resolves.toBeUndefined(); await expect( adapter.invalidate({ watermarkKey: "tracked:{id}:watermark", futureBufferMs: 100 }), ).resolves.toBeUndefined(); @@ -326,83 +306,54 @@ describe("Valkey GLIDE adapter", () => { expect(untrackedFrame[0]).toBe(1); expect(untrackedFrame[9]).toBe(0); expect(untrackedFrame.subarray(10).toString("utf8")).toBe("hello"); - const createdAtMs = Number(untrackedFrame.readBigUInt64BE(1)); - expect(createdAtMs).toBeGreaterThanOrEqual(before); - expect(createdAtMs).toBeLessThanOrEqual(after); + expect(Number(untrackedFrame.readBigUInt64BE(1))).toBe(1_234); expect(untrackedOptions).toEqual({ decoder: decoderBytes }); - expect(batchInstances).toHaveLength(1); - const trackedBatch = batchInstances[0]; - expect(trackedBatch?.isAtomic).toBe(false); - expect(trackedBatch?.commands).toHaveLength(2); - const [trackedSet, stamp] = trackedBatch?.commands ?? []; - expect(trackedSet?.[0]).toBe("SET"); - expect(trackedSet?.[1]).toBe("tracked:{id}:value"); - expect(trackedSet?.[3]).toBe("PX"); - expect(trackedSet?.[4]).toBe("2000"); - const trackedFrame = trackedSet?.[2] as Buffer; - expect(trackedFrame[0]).toBe(0); + const [trackedSet, trackedOptions] = client.customCommand.mock.calls[1] + ?? [[], undefined]; + expect(trackedSet[0]).toBe("SET"); + expect(trackedSet[1]).toBe("tracked:{id}:value"); + expect(trackedSet[3]).toBe("PX"); + expect(trackedSet[4]).toBe("2000"); + const trackedFrame = trackedSet[2] as Buffer; + expect(trackedFrame[0]).toBe(1); expect(trackedFrame[9]).toBe(1); expect(trackedFrame.subarray(10)).toEqual(binary); - const nonce = trackedFrame.subarray(1, 9); - expect(stamp).toEqual([ - "EVALSHA", - createHash("sha1").update(WRITE_TRACKED_STAMP_SCRIPT).digest("hex"), - "2", - "tracked:{id}:value", - "tracked:{id}:watermark", - "2000", - nonce, - ]); - expect(client.exec).toHaveBeenCalledTimes(1); - expect(client.exec).toHaveBeenCalledWith(trackedBatch, false, { decoder: decoderBytes }); + expect(Number(trackedFrame.readBigUInt64BE(1))).toBe(2_345); + expect(trackedOptions).toEqual({ decoder: decoderBytes }); - // Call 1 is the untracked SET; invalidation dispatches by its source SHA1. - expect(client.customCommand).toHaveBeenCalledTimes(2); + expect(batchInstances).toHaveLength(0); + expect(client.exec).not.toHaveBeenCalled(); + expect(client.customCommand).toHaveBeenCalledTimes(3); expect(client.customCommand).toHaveBeenNthCalledWith( - 2, + 3, [ "EVALSHA", createHash("sha1").update(INVALIDATE_CACHE_SCRIPT).digest("hex"), "1", "tracked:{id}:watermark", "100", + "3456", ], { decoder: decoderBytes }, ); - }); - - it("fails a tracked write whose placeholder was lost before the stamp", async () => { - const client = fakeClient([Buffer.from("OK"), 2]); - const adapter = createValkeyGlideDialCacheClient(client, mockGlide); - - const write = adapter.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", - cacheTtlMs: 1_000, - value: "tracked", - }); - await expect(write).rejects.toThrow("DialCache tracked write lost its placeholder before the stamp"); - await expect(write).rejects.toBeInstanceOf(DialCacheRedisPlaceholderLostError); - // Reply 2 is a settled outcome, not a recovery trigger. - expect(client.customCommand).not.toHaveBeenCalled(); + expect(now).toHaveBeenCalledTimes(3); }); it("routes cluster writes and invalidations to the slot primary", async () => { - const client = fakeClusterClient("OK", ["OK", 1], 1); + const client = fakeClusterClient("OK", "OK", 1); const adapter = createValkeyGlideDialCacheClient(client, mockGlide); await expect( adapter.write({ valueKey: "plain:value", cacheTtlMs: 1_000, value: "plain" }), - ).resolves.toBe(true); + ).resolves.toBeUndefined(); await expect( adapter.write({ valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", cacheTtlMs: 1_000, value: "tracked", }), - ).resolves.toBe(true); + ).resolves.toBeUndefined(); await expect( adapter.invalidate({ watermarkKey: "tracked:{id}:watermark", futureBufferMs: 25 }), ).resolves.toBeUndefined(); @@ -412,172 +363,87 @@ describe("Valkey GLIDE adapter", () => { decoder: decoderBytes, route: { type: "primarySlotKey", key: "plain:value" }, }); - expect(clusterBatchInstances).toHaveLength(1); - expect(client.exec).toHaveBeenCalledWith(clusterBatchInstances[0], false, { + const [, trackedOptions] = client.customCommand.mock.calls[1] ?? [[], undefined]; + expect(trackedOptions).toEqual({ decoder: decoderBytes, route: { type: "primarySlotKey", key: "tracked:{id}:value" }, }); - const [, invalidateOptions] = client.customCommand.mock.calls[1] ?? [[], undefined]; + expect(batchInstances).toHaveLength(0); + expect(client.exec).not.toHaveBeenCalled(); + const [, invalidateOptions] = client.customCommand.mock.calls[2] ?? [[], undefined]; expect(invalidateOptions).toEqual({ decoder: decoderBytes, route: { type: "primarySlotKey", key: "tracked:{id}:watermark" }, }); }); - it("routes the EVAL recovery to the slot primary on cluster", async () => { - const noscript = new Error("NOSCRIPT No matching script. Please use EVAL."); - const client = fakeClusterClient([Buffer.from("OK"), noscript], 1); + it("rejects out-of-range cacheTtlMs before dispatch and ceils fractional TTLs", async () => { + const client = fakeClient("OK"); const adapter = createValkeyGlideDialCacheClient(client, mockGlide); + const invalidTtls = [0, -1, Number.NaN, Number.POSITIVE_INFINITY, 31_536_000_001, "500" as unknown as number]; + for (const cacheTtlMs of invalidTtls) { + await expect( + adapter.write({ valueKey: "plain:value", cacheTtlMs, value: "plain" }), + ).rejects.toThrow(RangeError); + } + expect(client.customCommand).not.toHaveBeenCalled(); + expect(client.exec).not.toHaveBeenCalled(); + expect(batchInstances).toHaveLength(0); await expect(adapter.write({ valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", - cacheTtlMs: 2_000, + cacheTtlMs: 1_000.1, value: "tracked", - })).resolves.toBe(true); - - const trackedFrame = clusterBatchInstances[0]?.commands[0]?.[2] as Buffer; - expect(client.customCommand).toHaveBeenCalledWith( - [ - "EVAL", - WRITE_TRACKED_STAMP_SCRIPT, - "2", - "tracked:{id}:value", - "tracked:{id}:watermark", - "2000", - trackedFrame.subarray(1, 9), - ], - { - decoder: decoderBytes, - route: { type: "primarySlotKey", key: "tracked:{id}:value" }, - }, - ); - }); - - it("falls back to EVAL by source when the batched stamp hits NOSCRIPT", async () => { - const noscriptWordings = [ - // Raw server reply wording. - "NOSCRIPT No matching script. Please use EVAL.", - // GLIDE's mapped RequestError wording. - "An error was signalled by the server: - NoScriptError: No matching script.", - // Case drift must not blind the stamp's recovery either. - "noscript no matching script", - ]; - for (const wording of noscriptWordings) { - batchInstances.length = 0; - const client = fakeClient([Buffer.from("OK"), new Error(wording)], 1); - const adapter = createValkeyGlideDialCacheClient(client, mockGlide); - - await expect(adapter.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", - cacheTtlMs: 2_000, - value: "tracked", - })).resolves.toBe(true); - - const trackedFrame = batchInstances[0]?.commands[0]?.[2] as Buffer; - expect(client.customCommand).toHaveBeenCalledTimes(1); - expect(client.customCommand).toHaveBeenCalledWith( - [ - "EVAL", - WRITE_TRACKED_STAMP_SCRIPT, - "2", - "tracked:{id}:value", - "tracked:{id}:watermark", - "2000", - trackedFrame.subarray(1, 9), - ], - { decoder: decoderBytes }, - ); - } + })).resolves.toBeUndefined(); + expect(client.customCommand.mock.calls[0]?.[0]?.[4]).toBe("1001"); }); - it("rejects out-of-range cacheTtlMs before batching and ceils fractional TTLs", async () => { - const client = fakeClient([Buffer.from("OK"), 1]); + it("rejects invalid application timestamps before dispatching mutations", async () => { + const now = vi.spyOn(Date, "now"); + const client = fakeClient(); const adapter = createValkeyGlideDialCacheClient(client, mockGlide); - const invalidTtls = [0, -1, Number.NaN, Number.POSITIVE_INFINITY, 31_536_000_001, "500" as unknown as number]; - for (const cacheTtlMs of invalidTtls) { + + for (const timestampMs of [ + -1, + 1.5, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.MAX_SAFE_INTEGER + 1, + ]) { + now.mockReturnValue(timestampMs); await expect( - adapter.write({ valueKey: "plain:value", cacheTtlMs, value: "plain" }), + adapter.write({ valueKey: "plain:value", cacheTtlMs: 1_000, value: "plain" }), ).rejects.toThrow(RangeError); await expect( - adapter.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", - cacheTtlMs, - value: "tracked", - }), + adapter.invalidate({ watermarkKey: "tracked:{id}:watermark", futureBufferMs: 50 }), ).rejects.toThrow(RangeError); } + expect(client.customCommand).not.toHaveBeenCalled(); expect(client.exec).not.toHaveBeenCalled(); expect(batchInstances).toHaveLength(0); - - await expect(adapter.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", - cacheTtlMs: 1_000.1, - value: "tracked", - })).resolves.toBe(true); - const [trackedSet, stamp] = batchInstances[0]?.commands ?? []; - expect(trackedSet?.[4]).toBe("1001"); - expect(stamp?.[5]).toBe("1001"); - expect(Buffer.isBuffer(stamp?.[6])).toBe(true); }); - it("surfaces batched SET and stamp command errors", async () => { + it("surfaces SET command errors", async () => { const setFailure = new Error("OOM command not allowed when used memory > 'maxmemory'."); - const setClient = fakeClient([setFailure, 1]); + const setClient = fakeClient(); + setClient.customCommand.mockRejectedValueOnce(setFailure); const setAdapter = createValkeyGlideDialCacheClient(setClient, mockGlide); await expect(setAdapter.write({ valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", cacheTtlMs: 1_000, value: "tracked", })).rejects.toBe(setFailure); - expect(setClient.customCommand).not.toHaveBeenCalled(); - - const stampFailure = new Error("ERR invalid DialCache watermark"); - const stampClient = fakeClient([Buffer.from("OK"), stampFailure]); - const stampAdapter = createValkeyGlideDialCacheClient(stampClient, mockGlide); - await expect(stampAdapter.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", - cacheTtlMs: 1_000, - value: "tracked", - })).rejects.toBe(stampFailure); - expect(stampClient.customCommand).not.toHaveBeenCalled(); + expect(setClient.customCommand).toHaveBeenCalledTimes(1); }); - it("validates write batch envelopes and SET replies", async () => { - const envelopeClient = fakeClient("not-a-batch-reply"); - const envelopeAdapter = createValkeyGlideDialCacheClient(envelopeClient, mockGlide); - await expect(envelopeAdapter.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", - cacheTtlMs: 1_000, - value: "tracked", - })).rejects.toBeInstanceOf(DialCacheRedisPayloadError); - + it("validates native SET replies", async () => { const setReplyClient = fakeClient("QUEUED"); const setReplyAdapter = createValkeyGlideDialCacheClient(setReplyClient, mockGlide); await expectProtocolError( Promise.resolve(setReplyAdapter.write({ valueKey: "plain:value", cacheTtlMs: 1_000, value: "plain" })), "Invalid DialCache Redis SET reply; expected OK", ); - - // A bad SET reply wins over a failing stamp, matching the write contract. - const combinedClient = fakeClient(["QUEUED", new Error("ERR invalid DialCache watermark")]); - const combinedAdapter = createValkeyGlideDialCacheClient(combinedClient, mockGlide); - await expectProtocolError( - Promise.resolve(combinedAdapter.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", - cacheTtlMs: 1_000, - value: "tracked", - })), - "Invalid DialCache Redis SET reply; expected OK", - ); }); it("rejects malformed native read and mutation script replies", async () => { @@ -586,8 +452,8 @@ describe("Valkey GLIDE adapter", () => { redisFrame("invalid", { encoding: 2 }), "not-a-batch-reply", [[redisFrame("missing-watermark")]], - [Buffer.from("OK"), "not-an-integer"], - null, + "QUEUED", + 0, ); const adapter = createValkeyGlideDialCacheClient(client, mockGlide); @@ -604,11 +470,10 @@ describe("Valkey GLIDE adapter", () => { await expectProtocolError( Promise.resolve(adapter.write({ valueKey: "bad-write:{id}:value", - watermarkKey: "bad-write:{id}:watermark", cacheTtlMs: 1_000, value: "value", })), - "Invalid DialCache Redis write reply; expected integer 0, 1, or 2", + "Invalid DialCache Redis SET reply; expected OK", ); await expectProtocolError( Promise.resolve( @@ -618,26 +483,9 @@ describe("Valkey GLIDE adapter", () => { ); }); - it("rejects every out-of-domain write and invalidation reply", async () => { - const writeMessage = "Invalid DialCache Redis write reply; expected integer 0, 1, or 2"; + it("rejects every out-of-domain invalidation reply", async () => { const invalidationMessage = "Invalid DialCache Redis invalidate reply; expected integer 1"; - for (const reply of INVALID_WRITE_REPLIES) { - const tracked = createValkeyGlideDialCacheClient( - fakeClient([Buffer.from("OK"), reply]), - mockGlide, - ); - await expectProtocolError( - Promise.resolve(tracked.write({ - valueKey: "tracked:{id}:value", - watermarkKey: "tracked:{id}:watermark", - cacheTtlMs: 1_000, - value: "tracked", - })), - writeMessage, - ); - } - for (const reply of INVALID_INVALIDATION_REPLIES) { const client = fakeClient(reply); const adapter = createValkeyGlideDialCacheClient(client, mockGlide); @@ -654,6 +502,7 @@ describe("Valkey GLIDE adapter", () => { }); it("retries any invalidation rejection once with EVAL by source", async () => { + const now = vi.spyOn(Date, "now").mockReturnValue(1_234); // NOSCRIPT is the common trigger, but the retry deliberately covers every // rejection: the invalidation script is idempotent, and an // EVALSHA-rejecting proxy must self-heal rather than fail every call. @@ -670,12 +519,25 @@ describe("Valkey GLIDE adapter", () => { ).resolves.toBeUndefined(); expect(client.customCommand).toHaveBeenCalledTimes(2); + expect(client.customCommand).toHaveBeenNthCalledWith( + 1, + [ + "EVALSHA", + createHash("sha1").update(INVALIDATE_CACHE_SCRIPT).digest("hex"), + "1", + "tracked:{id}:watermark", + "50", + "1234", + ], + { decoder: decoderBytes }, + ); expect(client.customCommand).toHaveBeenNthCalledWith( 2, - ["EVAL", INVALIDATE_CACHE_SCRIPT, "1", "tracked:{id}:watermark", "50"], + ["EVAL", INVALIDATE_CACHE_SCRIPT, "1", "tracked:{id}:watermark", "50", "1234"], { decoder: decoderBytes }, ); } + expect(now).toHaveBeenCalledTimes(2); }); it("chains the original rejection when the invalidation retry also fails", async () => { @@ -750,7 +612,7 @@ describe("Valkey GLIDE adapter", () => { }; const client = fakeClient( [[redisFrame("tracked"), Buffer.from("0")]], - [Buffer.from("OK"), 1], + "OK", 1, ); const adapter = createValkeyGlideDialCacheClient(client, mockGlide); @@ -761,19 +623,16 @@ describe("Valkey GLIDE adapter", () => { }); await adapter.write({ valueKey: "module:{instance}:value", - watermarkKey: "module:{instance}:watermark", cacheTtlMs: 1_000, value: "value", }); await adapter.invalidate({ watermarkKey: "module:{instance}:watermark", futureBufferMs: 5 }); const [readBatch, , readOptions] = client.exec.mock.calls[0] ?? []; - const [writeBatch, , writeOptions] = client.exec.mock.calls[1] ?? []; - const [, invalidateOptions] = client.customCommand.mock.calls[0] ?? []; + const [, writeOptions] = client.customCommand.mock.calls[0] ?? []; + const [, invalidateOptions] = client.customCommand.mock.calls[1] ?? []; expect(readBatch).toBeInstanceOf(MockBatch); expect(readBatch).not.toBeInstanceOf(otherGlide.Batch); - expect(writeBatch).toBeInstanceOf(MockBatch); - expect(writeBatch).not.toBeInstanceOf(otherGlide.Batch); expect(readOptions?.decoder).toBe(mockGlide.Decoder.Bytes); expect(readOptions?.decoder).not.toBe(otherGlide.Decoder.Bytes); expect(writeOptions?.decoder).toBe(mockGlide.Decoder.Bytes);