From 390bc1f0fb97ae9b514ec519ee170ef583193b00 Mon Sep 17 00:00:00 2001 From: Jamie Holding Date: Mon, 28 Sep 2026 21:46:47 +0100 Subject: [PATCH 1/7] feat(history): changeRows() exposes the opening state; span().finalThrough changeRows() flattened the raw history envelope to { entityId, row } and threw away `opening`, the state each entity was in at the start of the range. Without it the stretch before an entity's first change has no known status, which on a night a ride runs past midnight is real operating time: Space Mountain on 2026-09-26 loses 63 seconds and no longer agrees with its own daily row. The result is still the same async generator, so `for await` is unchanged. It now also carries `opening`, keyed by entity id, readable once the response has arrived (after iterating, or after `await changes.load()`), for one request. HistorySpan gains `finalThrough`: the earlier of recordedTo and retrievableThrough, the newest day whose daily rows will not change again. The Space Mountain captures are shared byte for byte with the Python SDK. Co-Authored-By: Claude Opus 5.5 (1M context) --- .prettierignore | 2 + src/ergonomic/history.ts | 141 +- src/index.ts | 2 + test/fixtures/README.md | 15 + .../space_mountain_daily_2026-09-26.json | 47 + .../space_mountain_history_2026-09-26.json | 3356 +++++++++++++++++ test/unit/history-opening.test.ts | 233 ++ test/unit/history-paging.test.ts | 36 + 8 files changed, 3814 insertions(+), 18 deletions(-) create mode 100644 test/fixtures/space_mountain_daily_2026-09-26.json create mode 100644 test/fixtures/space_mountain_history_2026-09-26.json create mode 100644 test/unit/history-opening.test.ts diff --git a/.prettierignore b/.prettierignore index bb82e8c..6773a6b 100644 --- a/.prettierignore +++ b/.prettierignore @@ -4,3 +4,5 @@ docs-site/ coverage/ src/_generated/ *.log +# Shared byte for byte with the Python SDK, which tests against the same capture. +test/fixtures/space_mountain_*.json diff --git a/src/ergonomic/history.ts b/src/ergonomic/history.ts index 608fe4e..0f0aee7 100644 --- a/src/ergonomic/history.ts +++ b/src/ergonomic/history.ts @@ -32,6 +32,12 @@ import type { components } from '../_generated/schema'; type HistoryDailyRow = components['schemas']['HistoryDailyRow']; type HistoryRow = components['schemas']['HistoryRow']; +/** + * The state an entity was in at the START of a raw history range, before its + * first change: the same shape as a row, plus when it was last observed. + */ +export type HistoryOpening = components['schemas']['HistoryOpening']; + /** * The hourly history budget is spent and the wait is longer than this client * will sit through. @@ -111,20 +117,41 @@ export interface HistorySpan { * run ends in 403s. */ retrievableThrough: string | null; + /** + * The newest day whose daily rows will not change again, or null. + * + * `retrievableThrough` is usually today, and today's row is the day so far. + * Recent days can still change after that too: the archive records days 2 to + * 3 behind live data. `recordedTo` is the newest day the archive holds, so a + * day on or before it is final. Store those, and ask for anything later again + * once `finalThrough` has moved past it. + * + * The earlier of `recordedTo` and `retrievableThrough`, because a key may be + * entitled to fewer days than the archive holds. Null when either is unknown. + */ + finalThrough: string | null; +} + +/** The earlier of the two days, or null when either is unknown. ISO days sort as text. */ +function earlierDay(a: string | null, b: string | null): string | null { + if (a === null || b === null) return null; + return a < b ? a : b; } function toSpan(document: EntityHistoryCoverage): HistorySpan { - if ('summary' in document) { - return { - archiveFrom: document.summary.archiveFrom, - recordedTo: document.summary.recordedTo, - retrievableThrough: document.summary.retrievableThrough, - }; - } + const [archiveFrom, recordedTo, retrievableThrough] = + 'summary' in document + ? [ + document.summary.archiveFrom, + document.summary.recordedTo, + document.summary.retrievableThrough, + ] + : [document.firstRecordedAt, document.lastRecordedAt, document.retrievableThrough]; return { - archiveFrom: document.firstRecordedAt, - recordedTo: document.lastRecordedAt, - retrievableThrough: document.retrievableThrough, + archiveFrom, + recordedTo, + retrievableThrough, + finalThrough: earlierDay(recordedTo, retrievableThrough), }; } @@ -161,6 +188,43 @@ function* changeEntries(envelope: EntityHistory): Generator { for (const row of envelope.history) yield { entityId: envelope.id, row }; } +/** Each entity's `opening`, keyed by id, in the order the response lists them. */ +function openingsOf(envelope: EntityHistory): Record { + if ('entities' in envelope) { + const out: Record = {}; + for (const entity of envelope.entities) out[entity.id] = entity.opening; + return out; + } + return { [envelope.id]: envelope.opening }; +} + +/** + * What {@link HistoryApi.changeRows} returns: the rows, and the state before them. + * + * It is the async generator it always was. `for await` yields the same + * `{ entityId, row }` entries, and each row is the entity's complete live data + * from its `time` until the next row's. + * + * `opening` is the one thing the rows cannot tell you: the state in force at the + * START of the range, before the first change. Without it, the stretch between + * midnight and an entity's first change has no known status. On a night a ride + * runs past midnight that is real operating time, and a day rebuilt from the + * rows alone disagrees with the daily summary. It has an entry for every entity + * in the response, including one that did not change at all that day. + * + * Nothing is requested until the result is first used, as before. A getter + * cannot await, so `opening` is readable once the response has arrived: after + * the first step of iteration, or straight away with + * `const changes = await history.changeRows(query).load()`. Either way it is + * one request. + */ +export type HistoryChanges = AsyncGenerator & { + /** The state at the start of the range, per entity id. Throws before the response. */ + readonly opening: Record; + /** Make the request now, if it has not been made, and resolve to this object. */ + load(): Promise; +}; + export interface BudgetOptions { /** Past this, a 429 becomes {@link BudgetExhaustedError} instead of a retry. */ maxWaitMs?: number; @@ -273,20 +337,61 @@ export class HistoryApi { } /** - * Every recorded change in the range, flattened to one stream. + * Every recorded change in the range, flattened to one stream, plus the state + * before the first of them. * * A park answers one day per call; a single entity answers up to 31 days. * The caller does not have to know which cap applies: ask for what you want, * and the API answers or says the range is too long. + * + * Iterate the result for the rows. Its `opening` is each entity's state at the + * start of the range, which is what a day has to be rebuilt from. See + * {@link HistoryChanges}. */ - async *changeRows(options: ChangesOptions = {}): AsyncGenerator { + changeRows(options: ChangesOptions = {}): HistoryChanges { const maxWaitMs = options.maxWaitMs ?? DEFAULT_MAX_WAIT_MS; - let envelope: EntityHistory; - try { - envelope = await this.raw.getEntityHistory(this.entityId, toQuery(options)); - } catch (error) { - throw asBudgetError(error, maxWaitMs); + // ONE promise, shared, so `load()` and the iteration (or two `load()`s in + // flight) cost one request between them. + let pending: Promise | null = null; + let envelope: EntityHistory | null = null; + const fetchOnce = (): Promise => { + pending ??= this.raw.getEntityHistory(this.entityId, toQuery(options)).then( + (value) => { + envelope = value; + return value; + }, + (error: unknown) => { + throw asBudgetError(error, maxWaitMs); + }, + ); + return pending; + }; + + async function* rows(): AsyncGenerator { + yield* changeEntries(await fetchOnce()); } - yield* changeEntries(envelope); + + const changes = rows() as HistoryChanges; + Object.defineProperties(changes, { + opening: { + enumerable: true, + get(): Record { + if (envelope === null) { + throw new Error( + 'the response has not arrived yet: iterate first, or ' + + '`await changes.load()` before reading `opening`', + ); + } + return openingsOf(envelope); + }, + }, + load: { + value: async (): Promise => { + await fetchOnce(); + return changes; + }, + }, + }); + return changes; } } diff --git a/src/index.ts b/src/index.ts index 21d4cf9..39b1b89 100644 --- a/src/index.ts +++ b/src/index.ts @@ -30,6 +30,8 @@ export { type ChangesOptions, type DailyEntry, type DaysOptions, + type HistoryChanges, + type HistoryOpening, // `HistoryPage` is what `onPage` hands you, so a caller who wants to name the // type or store a page needs it. It was declared `export` in its own module and // never re-exported here, so `import type { HistoryPage } from 'themeparks'` diff --git a/test/fixtures/README.md b/test/fixtures/README.md index 2c0cbd5..cb6edbe 100644 --- a/test/fixtures/README.md +++ b/test/fixtures/README.md @@ -25,3 +25,18 @@ rows after 2026-08-30 — so a checkpoint taken from the newest ROW rewinds and re-downloads days already written. On the full 72-entity capture the same page holds 1,684 rows, of which every one carries `unknownMinutes` and an `inParkHours` block that the published schema does not mention. + +## space_mountain_history_2026-09-26.json / space_mountain_daily_2026-09-26.json + +`GET /entity/b2260923-9315-40fd-9c6b-44dd811dbe64/history?date=2026-09-26` and +`GET /entity/b2260923-9315-40fd-9c6b-44dd811dbe64/history/daily?date=2026-09-26`, +both captured 2026-09-28 without a key, verbatim. The same bytes are in the +Python SDK, so `.prettierignore` leaves them alone. + +They are the oracle for `changeRows().opening`. Space Mountain's opening that +day is `OPERATING`, because the previous night's hours ran past midnight, and its +first row is the close at 00:01:03 local. A day rebuilt from the rows alone has +63 seconds with no known status; rebuilt from the opening plus the rows it has +none, and its first open and last close are the daily row's `firstOperatingAt` +and `lastClosedAt`. If a re-capture picks a day whose opening is `CLOSED`, the +tests stop being able to tell the two apart, and one of them says so. diff --git a/test/fixtures/space_mountain_daily_2026-09-26.json b/test/fixtures/space_mountain_daily_2026-09-26.json new file mode 100644 index 0000000..bc8612d --- /dev/null +++ b/test/fixtures/space_mountain_daily_2026-09-26.json @@ -0,0 +1,47 @@ +{ + "id": "b2260923-9315-40fd-9c6b-44dd811dbe64", + "name": "Space Mountain", + "entityType": "ATTRACTION", + "parentId": "75ea578a-adc8-4116-a54d-dccb60765ef9", + "destinationId": "e957da41-3552-4cf6-b636-5babc5cbc4e5", + "timezone": "America/New_York", + "range": { + "from": "2026-09-26", + "to": "2026-09-26" + }, + "coverage": { + "firstRecordedAt": "2021-07-03" + }, + "days": [ + { + "date": "2026-09-26", + "firstOperatingAt": "2026-09-26T11:30:53Z", + "lastClosedAt": "2026-09-27T03:01:04Z", + "operatingMinutes": 929, + "downMinutes": 0, + "unknownMinutes": 4, + "standby": { + "min": 5, + "p50": 40, + "mean": 37, + "p90": 55, + "max": 65 + }, + "inParkHours": { + "scheduledMinutes": 930, + "operatingMinutes": 929, + "downMinutes": 0, + "unknownMinutes": 0, + "standby": { + "min": 5, + "p50": 40, + "mean": 37, + "p90": 55, + "max": 65 + } + }, + "changes": 187 + } + ], + "next": null +} \ No newline at end of file diff --git a/test/fixtures/space_mountain_history_2026-09-26.json b/test/fixtures/space_mountain_history_2026-09-26.json new file mode 100644 index 0000000..f0ee8f7 --- /dev/null +++ b/test/fixtures/space_mountain_history_2026-09-26.json @@ -0,0 +1,3356 @@ +{ + "id": "b2260923-9315-40fd-9c6b-44dd811dbe64", + "name": "Space Mountain", + "entityType": "ATTRACTION", + "parentId": "75ea578a-adc8-4116-a54d-dccb60765ef9", + "destinationId": "e957da41-3552-4cf6-b636-5babc5cbc4e5", + "timezone": "America/New_York", + "range": { + "from": "2026-09-26", + "to": "2026-09-26" + }, + "coverage": { + "firstRecordedAt": "2021-07-03" + }, + "opening": { + "time": "2026-09-26T04:00:00Z", + "observedAt": "2026-09-26T03:56:53Z", + "observedAtByKind": { + "status": "2026-09-26T03:56:53Z", + "queue.STANDBY": "2026-09-26T03:56:53Z", + "queue.RETURN_TIME": "2026-09-26T03:56:53Z" + }, + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 15 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + "history": [ + { + "time": "2026-09-26T04:01:03Z", + "changed": [ + "status", + "queue.STANDBY.waitTime" + ], + "status": "CLOSED", + "queue": { + "STANDBY": { + "waitTime": null + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-26T04:04:53Z", + "changed": [ + "queue.RETURN_TIME.state", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "CLOSED", + "queue": { + "STANDBY": { + "waitTime": null + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T08:10:00-04:00", + "returnEnd": "2026-09-26T09:10:00-04:00" + } + } + }, + { + "time": "2026-09-26T11:30:53Z", + "changed": [ + "status", + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 10 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T08:10:00-04:00", + "returnEnd": "2026-09-26T09:10:00-04:00" + } + } + }, + { + "time": "2026-09-26T11:31:58Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 5 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T08:10:00-04:00", + "returnEnd": "2026-09-26T09:10:00-04:00" + } + } + }, + { + "time": "2026-09-26T11:34:39Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 5 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T08:00:00-04:00", + "returnEnd": "2026-09-26T09:00:00-04:00" + } + } + }, + { + "time": "2026-09-26T11:55:45Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 5 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T09:05:00-04:00", + "returnEnd": "2026-09-26T10:05:00-04:00" + } + } + }, + { + "time": "2026-09-26T12:05:43Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 5 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T08:55:00-04:00", + "returnEnd": "2026-09-26T09:55:00-04:00" + } + } + }, + { + "time": "2026-09-26T12:09:16Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 5 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T08:10:00-04:00", + "returnEnd": "2026-09-26T09:10:00-04:00" + } + } + }, + { + "time": "2026-09-26T12:15:06Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 5 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T08:55:00-04:00", + "returnEnd": "2026-09-26T09:55:00-04:00" + } + } + }, + { + "time": "2026-09-26T12:21:18Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 5 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T08:40:00-04:00", + "returnEnd": "2026-09-26T09:40:00-04:00" + } + } + }, + { + "time": "2026-09-26T12:24:32Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 5 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T08:25:00-04:00", + "returnEnd": "2026-09-26T09:25:00-04:00" + } + } + }, + { + "time": "2026-09-26T12:30:01Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 5 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T08:35:00-04:00", + "returnEnd": "2026-09-26T09:35:00-04:00" + } + } + }, + { + "time": "2026-09-26T12:34:37Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 5 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T09:05:00-04:00", + "returnEnd": "2026-09-26T10:05:00-04:00" + } + } + }, + { + "time": "2026-09-26T12:39:54Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 5 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T09:00:00-04:00", + "returnEnd": "2026-09-26T10:00:00-04:00" + } + } + }, + { + "time": "2026-09-26T12:46:10Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 5 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T09:15:00-04:00", + "returnEnd": "2026-09-26T10:15:00-04:00" + } + } + }, + { + "time": "2026-09-26T12:55:30Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 5 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T09:20:00-04:00", + "returnEnd": "2026-09-26T10:20:00-04:00" + } + } + }, + { + "time": "2026-09-26T12:57:59Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 10 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T09:20:00-04:00", + "returnEnd": "2026-09-26T10:20:00-04:00" + } + } + }, + { + "time": "2026-09-26T13:04:51Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 10 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T09:25:00-04:00", + "returnEnd": "2026-09-26T10:25:00-04:00" + } + } + }, + { + "time": "2026-09-26T13:11:06Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 10 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T09:35:00-04:00", + "returnEnd": "2026-09-26T10:35:00-04:00" + } + } + }, + { + "time": "2026-09-26T13:15:14Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 10 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T09:30:00-04:00", + "returnEnd": "2026-09-26T10:30:00-04:00" + } + } + }, + { + "time": "2026-09-26T13:15:56Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 15 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T09:30:00-04:00", + "returnEnd": "2026-09-26T10:30:00-04:00" + } + } + }, + { + "time": "2026-09-26T13:18:01Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 20 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T09:30:00-04:00", + "returnEnd": "2026-09-26T10:30:00-04:00" + } + } + }, + { + "time": "2026-09-26T13:20:26Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 20 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T09:45:00-04:00", + "returnEnd": "2026-09-26T10:45:00-04:00" + } + } + }, + { + "time": "2026-09-26T13:24:44Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 20 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T09:55:00-04:00", + "returnEnd": "2026-09-26T10:55:00-04:00" + } + } + }, + { + "time": "2026-09-26T13:29:45Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 20 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T10:10:00-04:00", + "returnEnd": "2026-09-26T11:10:00-04:00" + } + } + }, + { + "time": "2026-09-26T13:34:42Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 20 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T10:20:00-04:00", + "returnEnd": "2026-09-26T11:20:00-04:00" + } + } + }, + { + "time": "2026-09-26T13:39:10Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 20 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T10:25:00-04:00", + "returnEnd": "2026-09-26T11:25:00-04:00" + } + } + }, + { + "time": "2026-09-26T13:45:23Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 20 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T10:35:00-04:00", + "returnEnd": "2026-09-26T11:35:00-04:00" + } + } + }, + { + "time": "2026-09-26T13:49:47Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 20 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T10:40:00-04:00", + "returnEnd": "2026-09-26T11:40:00-04:00" + } + } + }, + { + "time": "2026-09-26T13:54:23Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 20 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T10:45:00-04:00", + "returnEnd": "2026-09-26T11:45:00-04:00" + } + } + }, + { + "time": "2026-09-26T14:00:06Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 20 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T10:55:00-04:00", + "returnEnd": "2026-09-26T11:55:00-04:00" + } + } + }, + { + "time": "2026-09-26T14:06:00Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 20 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T11:00:00-04:00", + "returnEnd": "2026-09-26T12:00:00-04:00" + } + } + }, + { + "time": "2026-09-26T14:08:57Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 20 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T11:15:00-04:00", + "returnEnd": "2026-09-26T12:15:00-04:00" + } + } + }, + { + "time": "2026-09-26T14:15:28Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 20 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T11:20:00-04:00", + "returnEnd": "2026-09-26T12:20:00-04:00" + } + } + }, + { + "time": "2026-09-26T14:21:30Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 20 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T11:25:00-04:00", + "returnEnd": "2026-09-26T12:25:00-04:00" + } + } + }, + { + "time": "2026-09-26T14:22:03Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T11:25:00-04:00", + "returnEnd": "2026-09-26T12:25:00-04:00" + } + } + }, + { + "time": "2026-09-26T14:24:32Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T11:20:00-04:00", + "returnEnd": "2026-09-26T12:20:00-04:00" + } + } + }, + { + "time": "2026-09-26T14:30:41Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T11:45:00-04:00", + "returnEnd": "2026-09-26T12:45:00-04:00" + } + } + }, + { + "time": "2026-09-26T14:40:04Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T11:55:00-04:00", + "returnEnd": "2026-09-26T12:55:00-04:00" + } + } + }, + { + "time": "2026-09-26T14:43:04Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T11:55:00-04:00", + "returnEnd": "2026-09-26T12:55:00-04:00" + } + } + }, + { + "time": "2026-09-26T14:45:01Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T12:05:00-04:00", + "returnEnd": "2026-09-26T13:05:00-04:00" + } + } + }, + { + "time": "2026-09-26T14:49:27Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T12:15:00-04:00", + "returnEnd": "2026-09-26T13:15:00-04:00" + } + } + }, + { + "time": "2026-09-26T14:54:56Z", + "changed": [ + "queue.STANDBY.waitTime", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T12:20:00-04:00", + "returnEnd": "2026-09-26T13:20:00-04:00" + } + } + }, + { + "time": "2026-09-26T14:59:25Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T12:10:00-04:00", + "returnEnd": "2026-09-26T13:10:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:05:08Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T12:30:00-04:00", + "returnEnd": "2026-09-26T13:30:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:06:12Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 35 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T12:30:00-04:00", + "returnEnd": "2026-09-26T13:30:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:11:26Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 35 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T11:15:00-04:00", + "returnEnd": "2026-09-26T12:15:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:14:31Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 35 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T12:00:00-04:00", + "returnEnd": "2026-09-26T13:00:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:16:56Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T12:00:00-04:00", + "returnEnd": "2026-09-26T13:00:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:20:46Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T12:25:00-04:00", + "returnEnd": "2026-09-26T13:25:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:20:56Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T12:25:00-04:00", + "returnEnd": "2026-09-26T13:25:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:24:25Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T12:50:00-04:00", + "returnEnd": "2026-09-26T13:50:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:30:10Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T12:55:00-04:00", + "returnEnd": "2026-09-26T13:55:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:36:28Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T12:50:00-04:00", + "returnEnd": "2026-09-26T13:50:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:39:46Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T13:20:00-04:00", + "returnEnd": "2026-09-26T14:20:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:44:59Z", + "changed": [ + "queue.STANDBY.waitTime", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T13:35:00-04:00", + "returnEnd": "2026-09-26T14:35:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:49:12Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T13:50:00-04:00", + "returnEnd": "2026-09-26T14:50:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:54:04Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T13:50:00-04:00", + "returnEnd": "2026-09-26T14:50:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:54:08Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T12:40:00-04:00", + "returnEnd": "2026-09-26T13:40:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:58:12Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T12:40:00-04:00", + "returnEnd": "2026-09-26T13:40:00-04:00" + } + } + }, + { + "time": "2026-09-26T15:59:02Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T14:25:00-04:00", + "returnEnd": "2026-09-26T15:25:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:04:37Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T14:40:00-04:00", + "returnEnd": "2026-09-26T15:40:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:06:56Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T14:40:00-04:00", + "returnEnd": "2026-09-26T15:40:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:10:55Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T14:45:00-04:00", + "returnEnd": "2026-09-26T15:45:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:14:03Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T15:00:00-04:00", + "returnEnd": "2026-09-26T16:00:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:17:56Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T15:00:00-04:00", + "returnEnd": "2026-09-26T16:00:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:20:16Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T14:45:00-04:00", + "returnEnd": "2026-09-26T15:45:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:26:36Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T15:00:00-04:00", + "returnEnd": "2026-09-26T16:00:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:27:05Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 60 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T15:00:00-04:00", + "returnEnd": "2026-09-26T16:00:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:29:47Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 60 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T15:20:00-04:00", + "returnEnd": "2026-09-26T16:20:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:34:56Z", + "changed": [ + "queue.STANDBY.waitTime", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T15:30:00-04:00", + "returnEnd": "2026-09-26T16:30:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:36:03Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 60 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T15:30:00-04:00", + "returnEnd": "2026-09-26T16:30:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:39:09Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 60 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T15:40:00-04:00", + "returnEnd": "2026-09-26T16:40:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:42:19Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T15:40:00-04:00", + "returnEnd": "2026-09-26T16:40:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:44:58Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T15:50:00-04:00", + "returnEnd": "2026-09-26T16:50:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:46:56Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 60 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T15:50:00-04:00", + "returnEnd": "2026-09-26T16:50:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:50:58Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 60 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T15:35:00-04:00", + "returnEnd": "2026-09-26T16:35:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:52:55Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T15:35:00-04:00", + "returnEnd": "2026-09-26T16:35:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:58:15Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T15:35:00-04:00", + "returnEnd": "2026-09-26T16:35:00-04:00" + } + } + }, + { + "time": "2026-09-26T16:59:59Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T14:30:00-04:00", + "returnEnd": "2026-09-26T15:30:00-04:00" + } + } + }, + { + "time": "2026-09-26T17:04:14Z", + "changed": [ + "queue.STANDBY.waitTime", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 35 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T14:40:00-04:00", + "returnEnd": "2026-09-26T15:40:00-04:00" + } + } + }, + { + "time": "2026-09-26T17:14:41Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 35 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T16:35:00-04:00", + "returnEnd": "2026-09-26T17:35:00-04:00" + } + } + }, + { + "time": "2026-09-26T17:20:52Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 35 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T16:55:00-04:00", + "returnEnd": "2026-09-26T17:55:00-04:00" + } + } + }, + { + "time": "2026-09-26T17:24:04Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 35 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T17:00:00-04:00", + "returnEnd": "2026-09-26T18:00:00-04:00" + } + } + }, + { + "time": "2026-09-26T17:26:24Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T17:00:00-04:00", + "returnEnd": "2026-09-26T18:00:00-04:00" + } + } + }, + { + "time": "2026-09-26T17:30:05Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T17:10:00-04:00", + "returnEnd": "2026-09-26T18:10:00-04:00" + } + } + }, + { + "time": "2026-09-26T17:33:11Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T17:20:00-04:00", + "returnEnd": "2026-09-26T18:20:00-04:00" + } + } + }, + { + "time": "2026-09-26T17:39:36Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T17:25:00-04:00", + "returnEnd": "2026-09-26T18:25:00-04:00" + } + } + }, + { + "time": "2026-09-26T17:45:34Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T17:40:00-04:00", + "returnEnd": "2026-09-26T18:40:00-04:00" + } + } + }, + { + "time": "2026-09-26T17:49:07Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T17:35:00-04:00", + "returnEnd": "2026-09-26T18:35:00-04:00" + } + } + }, + { + "time": "2026-09-26T17:55:03Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T17:55:00-04:00", + "returnEnd": "2026-09-26T18:55:00-04:00" + } + } + }, + { + "time": "2026-09-26T17:58:00Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T14:30:00-04:00", + "returnEnd": "2026-09-26T15:30:00-04:00" + } + } + }, + { + "time": "2026-09-26T18:04:15Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T18:20:00-04:00", + "returnEnd": "2026-09-26T19:20:00-04:00" + } + } + }, + { + "time": "2026-09-26T18:10:37Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T18:10:00-04:00", + "returnEnd": "2026-09-26T19:10:00-04:00" + } + } + }, + { + "time": "2026-09-26T18:12:58Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T18:10:00-04:00", + "returnEnd": "2026-09-26T19:10:00-04:00" + } + } + }, + { + "time": "2026-09-26T18:14:56Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T18:35:00-04:00", + "returnEnd": "2026-09-26T19:35:00-04:00" + } + } + }, + { + "time": "2026-09-26T18:19:58Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T18:45:00-04:00", + "returnEnd": "2026-09-26T19:45:00-04:00" + } + } + }, + { + "time": "2026-09-26T18:22:56Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T18:45:00-04:00", + "returnEnd": "2026-09-26T19:45:00-04:00" + } + } + }, + { + "time": "2026-09-26T18:24:54Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T18:55:00-04:00", + "returnEnd": "2026-09-26T19:55:00-04:00" + } + } + }, + { + "time": "2026-09-26T18:26:03Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T18:55:00-04:00", + "returnEnd": "2026-09-26T19:55:00-04:00" + } + } + }, + { + "time": "2026-09-26T18:29:14Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T19:05:00-04:00", + "returnEnd": "2026-09-26T20:05:00-04:00" + } + } + }, + { + "time": "2026-09-26T18:32:55Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T19:05:00-04:00", + "returnEnd": "2026-09-26T20:05:00-04:00" + } + } + }, + { + "time": "2026-09-26T18:35:36Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T19:15:00-04:00", + "returnEnd": "2026-09-26T20:15:00-04:00" + } + } + }, + { + "time": "2026-09-26T18:40:16Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T19:30:00-04:00", + "returnEnd": "2026-09-26T20:30:00-04:00" + } + } + }, + { + "time": "2026-09-26T18:44:59Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T19:20:00-04:00", + "returnEnd": "2026-09-26T20:20:00-04:00" + } + } + }, + { + "time": "2026-09-26T18:49:31Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T19:45:00-04:00", + "returnEnd": "2026-09-26T20:45:00-04:00" + } + } + }, + { + "time": "2026-09-26T18:54:39Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:00:00-04:00", + "returnEnd": "2026-09-26T21:00:00-04:00" + } + } + }, + { + "time": "2026-09-26T19:00:38Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T19:45:00-04:00", + "returnEnd": "2026-09-26T20:45:00-04:00" + } + } + }, + { + "time": "2026-09-26T19:05:21Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:10:00-04:00", + "returnEnd": "2026-09-26T21:10:00-04:00" + } + } + }, + { + "time": "2026-09-26T19:14:03Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:10:00-04:00", + "returnEnd": "2026-09-26T21:10:00-04:00" + } + } + }, + { + "time": "2026-09-26T19:18:09Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:25:00-04:00", + "returnEnd": "2026-09-26T21:25:00-04:00" + } + } + }, + { + "time": "2026-09-26T19:27:31Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:35:00-04:00", + "returnEnd": "2026-09-26T21:35:00-04:00" + } + } + }, + { + "time": "2026-09-26T19:34:03Z", + "changed": [ + "queue.STANDBY.waitTime", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:40:00-04:00", + "returnEnd": "2026-09-26T21:40:00-04:00" + } + } + }, + { + "time": "2026-09-26T19:47:33Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:45:00-04:00", + "returnEnd": "2026-09-26T21:45:00-04:00" + } + } + }, + { + "time": "2026-09-26T19:53:49Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:50:00-04:00", + "returnEnd": "2026-09-26T21:50:00-04:00" + } + } + }, + { + "time": "2026-09-26T19:54:58Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:50:00-04:00", + "returnEnd": "2026-09-26T21:50:00-04:00" + } + } + }, + { + "time": "2026-09-26T19:56:02Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:50:00-04:00", + "returnEnd": "2026-09-26T21:50:00-04:00" + } + } + }, + { + "time": "2026-09-26T19:58:26Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:45:00-04:00", + "returnEnd": "2026-09-26T21:45:00-04:00" + } + } + }, + { + "time": "2026-09-26T20:04:20Z", + "changed": [ + "queue.STANDBY.waitTime", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:40:00-04:00", + "returnEnd": "2026-09-26T21:40:00-04:00" + } + } + }, + { + "time": "2026-09-26T20:08:04Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:50:00-04:00", + "returnEnd": "2026-09-26T21:50:00-04:00" + } + } + }, + { + "time": "2026-09-26T20:14:23Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T21:20:00-04:00", + "returnEnd": "2026-09-26T22:20:00-04:00" + } + } + }, + { + "time": "2026-09-26T20:18:10Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T21:25:00-04:00", + "returnEnd": "2026-09-26T22:25:00-04:00" + } + } + }, + { + "time": "2026-09-26T20:20:05Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T21:25:00-04:00", + "returnEnd": "2026-09-26T22:25:00-04:00" + } + } + }, + { + "time": "2026-09-26T20:22:04Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T21:25:00-04:00", + "returnEnd": "2026-09-26T22:25:00-04:00" + } + } + }, + { + "time": "2026-09-26T20:23:59Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T21:35:00-04:00", + "returnEnd": "2026-09-26T22:35:00-04:00" + } + } + }, + { + "time": "2026-09-26T20:28:22Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T18:05:00-04:00", + "returnEnd": "2026-09-26T19:05:00-04:00" + } + } + }, + { + "time": "2026-09-26T20:28:58Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T18:05:00-04:00", + "returnEnd": "2026-09-26T19:05:00-04:00" + } + } + }, + { + "time": "2026-09-26T20:33:18Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T21:45:00-04:00", + "returnEnd": "2026-09-26T22:45:00-04:00" + } + } + }, + { + "time": "2026-09-26T20:38:41Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T21:35:00-04:00", + "returnEnd": "2026-09-26T22:35:00-04:00" + } + } + }, + { + "time": "2026-09-26T20:42:06Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T21:35:00-04:00", + "returnEnd": "2026-09-26T22:35:00-04:00" + } + } + }, + { + "time": "2026-09-26T20:43:22Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T21:55:00-04:00", + "returnEnd": "2026-09-26T22:55:00-04:00" + } + } + }, + { + "time": "2026-09-26T20:49:17Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T21:45:00-04:00", + "returnEnd": "2026-09-26T22:45:00-04:00" + } + } + }, + { + "time": "2026-09-26T20:52:45Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:00:00-04:00", + "returnEnd": "2026-09-26T23:00:00-04:00" + } + } + }, + { + "time": "2026-09-26T20:59:12Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T19:00:00-04:00", + "returnEnd": "2026-09-26T20:00:00-04:00" + } + } + }, + { + "time": "2026-09-26T21:02:57Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T19:00:00-04:00", + "returnEnd": "2026-09-26T20:00:00-04:00" + } + } + }, + { + "time": "2026-09-26T21:05:12Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:00:00-04:00", + "returnEnd": "2026-09-26T23:00:00-04:00" + } + } + }, + { + "time": "2026-09-26T21:09:10Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:05:00-04:00", + "returnEnd": "2026-09-26T23:05:00-04:00" + } + } + }, + { + "time": "2026-09-26T21:14:54Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:10:00-04:00", + "returnEnd": "2026-09-26T23:10:00-04:00" + } + } + }, + { + "time": "2026-09-26T21:16:57Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:10:00-04:00", + "returnEnd": "2026-09-26T23:10:00-04:00" + } + } + }, + { + "time": "2026-09-26T21:18:33Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T21:20:00-04:00", + "returnEnd": "2026-09-26T22:20:00-04:00" + } + } + }, + { + "time": "2026-09-26T21:24:04Z", + "changed": [ + "queue.STANDBY.waitTime", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:15:00-04:00", + "returnEnd": "2026-09-26T23:15:00-04:00" + } + } + }, + { + "time": "2026-09-26T21:24:58Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:15:00-04:00", + "returnEnd": "2026-09-26T23:15:00-04:00" + } + } + }, + { + "time": "2026-09-26T21:28:59Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:15:00-04:00", + "returnEnd": "2026-09-26T23:15:00-04:00" + } + } + }, + { + "time": "2026-09-26T21:32:00Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:15:00-04:00", + "returnEnd": "2026-09-26T23:15:00-04:00" + } + } + }, + { + "time": "2026-09-26T21:34:07Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:20:00-04:00", + "returnEnd": "2026-09-26T23:20:00-04:00" + } + } + }, + { + "time": "2026-09-26T21:43:41Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:25:00-04:00", + "returnEnd": "2026-09-26T23:25:00-04:00" + } + } + }, + { + "time": "2026-09-26T21:48:32Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:20:00-04:00", + "returnEnd": "2026-09-26T23:20:00-04:00" + } + } + }, + { + "time": "2026-09-26T21:56:16Z", + "changed": [ + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:30:00-04:00", + "returnEnd": "2026-09-26T23:30:00-04:00" + } + } + }, + { + "time": "2026-09-26T21:59:25Z", + "changed": [ + "queue.RETURN_TIME.state", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-26T22:12:06Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 35 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-26T22:24:59Z", + "changed": [ + "queue.RETURN_TIME.state", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 35 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:45:00-04:00", + "returnEnd": "2026-09-26T21:45:00-04:00" + } + } + }, + { + "time": "2026-09-26T22:25:59Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:45:00-04:00", + "returnEnd": "2026-09-26T21:45:00-04:00" + } + } + }, + { + "time": "2026-09-26T22:26:58Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:45:00-04:00", + "returnEnd": "2026-09-26T21:45:00-04:00" + } + } + }, + { + "time": "2026-09-26T22:28:06Z", + "changed": [ + "queue.RETURN_TIME.state", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-26T22:33:04Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-26T22:34:06Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-26T22:36:06Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 60 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-26T22:43:01Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 65 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-26T22:53:06Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 60 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-26T22:54:10Z", + "changed": [ + "queue.RETURN_TIME.state", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 60 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T20:25:00-04:00", + "returnEnd": "2026-09-26T21:25:00-04:00" + } + } + }, + { + "time": "2026-09-26T22:57:46Z", + "changed": [ + "queue.RETURN_TIME.state", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 60 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-26T23:15:00Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 55 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-26T23:20:06Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-26T23:30:57Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-26T23:36:09Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T00:02:12Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T00:12:57Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T00:24:05Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 35 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T00:44:58Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 30 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T00:58:02Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 35 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T01:06:06Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T01:13:31Z", + "changed": [ + "queue.RETURN_TIME.state", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:10:00-04:00", + "returnEnd": "2026-09-26T23:10:00-04:00" + } + } + }, + { + "time": "2026-09-27T01:16:07Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 35 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:10:00-04:00", + "returnEnd": "2026-09-26T23:10:00-04:00" + } + } + }, + { + "time": "2026-09-27T01:19:12Z", + "changed": [ + "queue.RETURN_TIME.state", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 35 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T01:20:05Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 30 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T01:28:47Z", + "changed": [ + "queue.RETURN_TIME.state", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 30 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:05:00-04:00", + "returnEnd": "2026-09-26T23:05:00-04:00" + } + } + }, + { + "time": "2026-09-27T01:32:02Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 25 + }, + "RETURN_TIME": { + "state": "AVAILABLE", + "returnStart": "2026-09-26T22:05:00-04:00", + "returnEnd": "2026-09-26T23:05:00-04:00" + } + } + }, + { + "time": "2026-09-27T01:34:46Z", + "changed": [ + "queue.RETURN_TIME.state", + "queue.RETURN_TIME.returnStart", + "queue.RETURN_TIME.returnEnd" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 25 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T01:42:06Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 20 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T01:58:58Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 25 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T02:02:01Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 30 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T02:13:01Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 40 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T02:14:05Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T02:14:58Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 50 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T02:31:00Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 45 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T02:36:00Z", + "changed": [ + "queue.STANDBY.waitTime" + ], + "status": "OPERATING", + "queue": { + "STANDBY": { + "waitTime": 30 + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + }, + { + "time": "2026-09-27T03:01:04Z", + "changed": [ + "status", + "queue.STANDBY.waitTime" + ], + "status": "CLOSED", + "queue": { + "STANDBY": { + "waitTime": null + }, + "RETURN_TIME": { + "state": "FINISHED", + "returnStart": null, + "returnEnd": null + } + } + } + ], + "next": null +} \ No newline at end of file diff --git a/test/unit/history-opening.test.ts b/test/unit/history-opening.test.ts new file mode 100644 index 0000000..bbb98c7 --- /dev/null +++ b/test/unit/history-opening.test.ts @@ -0,0 +1,233 @@ +/** + * `changeRows()` hands back the `opening` state, so a day can be rebuilt from it. + * + * Every row of `/history` is the complete live data from its `time` until the + * next row. What the rows cannot say is the state BEFORE the first of them: that + * is the envelope's `opening` object, and `changeRows()` yielded the rows and + * threw the envelope away. A caller rebuilding a day then had no status for the + * seconds between the start of the range and the first change, which on a night + * a ride runs past midnight is real operating time. + * + * The oracle is a real capture shared with the Python SDK (see + * test/fixtures/README.md): Space Mountain on 2026-09-26, whose opening is + * OPERATING because the previous night's hours ran past midnight, and the daily + * summary the API computed for the same day. + */ + +import { describe, it, expect, vi } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { ThemeParks } from '../../src/client'; +import type { HistoryChanges, HistoryOpening } from '../../src/ergonomic/history'; +import type { FetchLike } from '../../src/transport'; + +interface Row { + time: string; + status?: string | null; +} + +const FIXTURES = resolve(__dirname, '../fixtures'); +const RAW = JSON.parse( + readFileSync(resolve(FIXTURES, 'space_mountain_history_2026-09-26.json'), 'utf8'), +) as { id: string; opening: HistoryOpening; history: Row[]; coverage: unknown }; +const DAILY = JSON.parse( + readFileSync(resolve(FIXTURES, 'space_mountain_daily_2026-09-26.json'), 'utf8'), +) as { days: { firstOperatingAt: string; lastClosedAt: string }[] }; +const SPACE_MOUNTAIN = RAW.id; + +function client(payload: unknown, seen: string[] = []) { + const fetchFn = vi.fn((url: unknown) => { + seen.push(String(url)); + return Promise.resolve( + new Response(JSON.stringify(payload), { headers: { 'content-type': 'application/json' } }), + ); + }); + return new ThemeParks({ fetch: fetchFn as unknown as FetchLike, cache: false }); +} + +async function collect(iterable: AsyncIterable): Promise { + const out: T[] = []; + for await (const item of iterable) out.push(item); + return out; +} + +/** + * (from, to, status) segments covering the day, the way a customer rebuilds one. + * Without an opening, the stretch before the first change has no known status. + */ +function timeline( + opening: HistoryOpening | null, + rows: Row[], + dayStart: number, + dayEnd: number, +): [number, number, string | null][] { + const segments: [number, number, string | null][] = []; + let cursor = dayStart; + let status: string | null = opening ? (opening.status ?? null) : null; + for (const row of rows) { + const at = Date.parse(row.time); + if (at > cursor) segments.push([cursor, at, status]); + cursor = Math.max(cursor, at); + status = row.status ?? null; + } + segments.push([cursor, dayEnd, status]); + return segments; +} + +const seconds = (segments: [number, number, string | null][], wanted: string | null) => + segments.filter(([, , s]) => s === wanted).reduce((sum, [a, b]) => sum + (b - a) / 1000, 0); + +describe('the capture is what this file says it is', () => { + it('opens OPERATING, carried over midnight, and its first row is a close', () => { + // Guard the oracle. If a re-capture picks a day whose opening is CLOSED, + // every test below passes with and without the fix. + expect(RAW.opening.status).toBe('OPERATING'); + expect(RAW.history[0]?.status).toBe('CLOSED'); + expect(RAW.history[0]!.time > RAW.opening.time).toBe(true); + }); +}); + +describe('changeRows() exposes opening', () => { + it('yields exactly the rows it always did', async () => { + const rows = await collect( + client(RAW).entity(SPACE_MOUNTAIN).history.changeRows({ date: '2026-09-26' }), + ); + expect(rows).toHaveLength(RAW.history.length); + expect(rows.every((r) => r.entityId === SPACE_MOUNTAIN)).toBe(true); + expect(Object.keys(rows[0]!)).toEqual(['entityId', 'row']); + expect(rows[0]?.row.status).toBe('CLOSED'); + }); + + it('keys opening by entity id, readable once iterated', async () => { + const changes = client(RAW).entity(SPACE_MOUNTAIN).history.changeRows({ date: '2026-09-26' }); + await collect(changes); + const opening = changes.opening[SPACE_MOUNTAIN]; + expect(opening?.status).toBe('OPERATING'); + expect(opening?.time).toBe('2026-09-26T04:00:00Z'); + expect(opening?.queue?.STANDBY?.waitTime).toBe(15); + }); + + it('load() fetches up front, and iterating after costs no second request', async () => { + const seen: string[] = []; + const changes = client(RAW, seen) + .entity(SPACE_MOUNTAIN) + .history.changeRows({ date: '2026-09-26' }); + expect(seen).toEqual([]); + const loaded = await changes.load(); + expect(loaded).toBe(changes); + expect(changes.opening[SPACE_MOUNTAIN]?.status).toBe('OPERATING'); + expect(await collect(changes)).toHaveLength(RAW.history.length); + expect(seen).toHaveLength(1); + }); + + it('two load() calls in flight share one request', async () => { + const seen: string[] = []; + const changes = client(RAW, seen) + .entity(SPACE_MOUNTAIN) + .history.changeRows({ date: '2026-09-26' }); + await Promise.all([changes.load(), changes.load()]); + expect(seen).toHaveLength(1); + }); + + it('says how to get opening when read before the response has arrived', () => { + const changes = client(RAW).entity(SPACE_MOUNTAIN).history.changeRows({ date: '2026-09-26' }); + expect(() => changes.opening).toThrow(/load\(\)/u); + }); + + it('is still an async generator', async () => { + const changes: AsyncGenerator = client(RAW) + .entity(SPACE_MOUNTAIN) + .history.changeRows({ date: '2026-09-26' }); + expect(changes[Symbol.asyncIterator]()).toBe(changes); + const first = await changes.next(); + expect(first.done).toBe(false); + expect(await changes.return(undefined)).toEqual({ done: true, value: undefined }); + }); + + it('gives a park one opening per entity, including one that did not change', async () => { + const park = { + id: 'park-1', + name: 'Magic Kingdom Park', + entityType: 'PARK', + timezone: 'America/New_York', + range: { from: '2026-09-26', to: '2026-09-26' }, + entities: [ + { + id: SPACE_MOUNTAIN, + name: 'Space Mountain', + entityType: 'ATTRACTION', + coverage: RAW.coverage, + opening: RAW.opening, + history: RAW.history.slice(0, 2), + }, + { + id: 'quiet-ride', + name: 'Nothing Changed Today', + entityType: 'ATTRACTION', + coverage: { firstRecordedAt: '2021-07-03' }, + opening: { time: '2026-09-26T04:00:00Z', status: 'REFURBISHMENT' }, + history: [], + }, + ], + next: null, + }; + const changes = client(park).entity('park-1').history.changeRows({ date: '2026-09-26' }); + const rows = await collect(changes); + expect(Object.keys(changes.opening)).toEqual([SPACE_MOUNTAIN, 'quiet-ride']); + // An entity with no rows at all is only knowable through its opening. + expect(changes.opening['quiet-ride']?.status).toBe('REFURBISHMENT'); + expect(rows.map((r) => r.entityId)).toEqual([SPACE_MOUNTAIN, SPACE_MOUNTAIN]); + }); + + it('a failed request rejects load() and the iteration alike', async () => { + const fetchFn = vi.fn(() => + Promise.resolve( + new Response(JSON.stringify({ error: { type: 'INVALID_RANGE' } }), { + status: 400, + headers: { 'content-type': 'application/json' }, + }), + ), + ); + const tp = new ThemeParks({ fetch: fetchFn as unknown as FetchLike, cache: false }); + const changes: HistoryChanges = tp.entity(SPACE_MOUNTAIN).history.changeRows({ date: 'x' }); + await expect(changes.load()).rejects.toThrow(); + await expect(collect(changes)).rejects.toThrow(); + }); +}); + +describe('a day rebuilds from opening plus the rows', () => { + // What opening is FOR, checked against the API's own daily row. + const DAY_START = Date.parse('2026-09-26T04:00:00Z'); // 00:00 EDT + const DAY_END = DAY_START + 86_400_000; + + async function rebuild(withOpening: boolean) { + const changes = client(RAW).entity(SPACE_MOUNTAIN).history.changeRows({ date: '2026-09-26' }); + const rows = (await collect(changes)).map((e) => e.row as Row); + const opening = withOpening ? (changes.opening[SPACE_MOUNTAIN] ?? null) : null; + return timeline(opening, rows, DAY_START, DAY_END); + } + + it('gives every second of the day a known status', async () => { + const segments = await rebuild(true); + expect(segments.reduce((sum, [a, b]) => sum + (b - a) / 1000, 0)).toBe(86_400); + expect(seconds(segments, null)).toBe(0); + }); + + it('without the opening, the first 63 seconds are unknown', async () => { + // The defect, measured: a ride OPERATING past midnight that a rebuild from + // rows alone cannot place. + expect(seconds(await rebuild(false), null)).toBe(63); + }); + + it('agrees with the daily row the API computed', async () => { + const segments = await rebuild(true); + const daily = DAILY.days[0]!; + const opened = segments.filter(([a, , s]) => s === 'OPERATING' && a > DAY_START); + const closed = segments.filter(([, b, s]) => s === 'OPERATING' && b < DAY_END); + const iso = (ms: number) => new Date(ms).toISOString().replace('.000Z', 'Z'); + expect(iso(opened[0]![0])).toBe(daily.firstOperatingAt); + expect(iso(closed.at(-1)![1])).toBe(daily.lastClosedAt); + // 63 s carried over midnight, then 11:30:53Z to 03:01:04Z. + expect(seconds(segments, 'OPERATING')).toBe(63 + 55_811); + }); +}); diff --git a/test/unit/history-paging.test.ts b/test/unit/history-paging.test.ts index ac1a7e3..6376492 100644 --- a/test/unit/history-paging.test.ts +++ b/test/unit/history-paging.test.ts @@ -165,6 +165,42 @@ describe('span()', () => { expect(span.recordedTo).toBe(summary.recordedTo); expect(span.retrievableThrough).toBe(summary.retrievableThrough); }); + + it('finalThrough is the earlier of recordedTo and retrievableThrough', async () => { + // retrievableThrough is usually today, whose row is the day so far, and the + // archive records days 2 to 3 behind. The real capture has recordedTo two + // days before retrievableThrough, so the fixture itself tells the two apart. + const park = await loadFixture('mk_history_coverage.json'); + const summary = park.summary as { recordedTo: string; retrievableThrough: string }; + expect(summary.recordedTo < summary.retrievableThrough).toBe(true); + const at = async (recordedTo: string | null, retrievableThrough: string | null) => + ( + await client( + vi.fn(() => + Promise.resolve( + json({ ...park, summary: { ...summary, recordedTo, retrievableThrough } }), + ), + ), + ) + .entity('mk') + .history.span() + ).finalThrough; + expect(await at(summary.recordedTo, summary.retrievableThrough)).toBe(summary.recordedTo); + // A key entitled to fewer days than the archive holds. + expect(await at('2026-09-21', '2026-09-01')).toBe('2026-09-01'); + expect(await at(null, '2026-09-23')).toBeNull(); + expect(await at('2026-09-21', null)).toBeNull(); + }); + + it('finalThrough reads an entity document too', async () => { + const entity = await loadFixture('mk_attraction_history_coverage.json'); + const span = await client(vi.fn(() => Promise.resolve(json(entity)))) + .entity('ride') + .history.span(); + const last = entity.lastRecordedAt as string; + const through = entity.retrievableThrough as string; + expect(span.finalThrough).toBe(last < through ? last : through); + }); }); describe('the hourly history budget', () => { From 436381400470a23467bc3e2f66626ad705bc644a Mon Sep 17 00:00:00 2001 From: Jamie Holding Date: Mon, 28 Sep 2026 22:02:36 +0100 Subject: [PATCH 2/7] fix(backfill): --since/--until, reruns add new days, only final days written Three defects from one customer-style run of 8.3.1, each reproduced against the live API first: 1. No date range. `--since` was an unknown option, so a key that reaches the whole archive downloaded all of it every time. `--since` and `--until` take a real calendar day as YYYY-MM-DD, both inclusive; since > until is refused before any request. A rerun accepts the same --since or a later one, and refuses one earlier than the day the file was asked to start from, or one that would leave a gap, saying what to do. 2. Reruns never updated. A finished park printed "already complete" and exited 0 without asking for a day. It now continues from the day after its last one; an interrupted run still resumes at its page boundary, and one interrupted before its first page records where it began. 3. Partial days, kept forever. A run ended at retrievableThrough, usually today. It now ends at span().finalThrough and says when it holds days back; the next run adds them once final. State moves to version 2: an 8.3 file has its last seven days before the old end removed (the rest byte for byte) and fetched again, and one wholly inside that window is downloaded again. Also: a finished state with a foreign column layout is refused instead of appended to; a state whose data file is gone starts again; and a continued file whose next day is older than the key may read is refused rather than jumping forward and leaving a gap. Eighteen new mutants cover these, and all 38 are killed. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/backfill.ts | 607 +++++++++++++++-- test/mutation/mutants.json | 126 ++++ test/unit/backfill-incremental.test.ts | 882 +++++++++++++++++++++++++ test/unit/backfill.test.ts | 67 +- 4 files changed, 1612 insertions(+), 70 deletions(-) create mode 100644 test/unit/backfill-incremental.test.ts diff --git a/src/backfill.ts b/src/backfill.ts index 38bce24..f0a0b8c 100644 --- a/src/backfill.ts +++ b/src/backfill.ts @@ -22,11 +22,22 @@ * full archive is before your window opens. So the first request is refused, * and the floor is read out of that 403. * - * 3. It records what it has written, so re-running continues an interrupted run - * and never appends a second copy of a finished one. + * 3. It records what it has written, so re-running is safe in every direction: + * an interrupted run continues at its page boundary, a FINISHED park is + * brought up to date from the day after its last one rather than appended to + * twice, and a file this command did not write is never touched without + * `--overwrite`. `(entityId, date)` is the natural key if you load blind: a + * run that died inside its first page resumes on the last day it wrote, so + * that one day can appear twice. * * 4. It takes names and destinations, not just park ids. A customer has * "Walt Disney World Resort", not four uuids. + * + * 5. It writes FINAL days only. Today's row is the day so far, and the archive + * records days 2 to 3 behind live data, so the newest days the API serves can + * still change. The run ends at `span().finalThrough`, the newest day the + * archive holds, and the next run carries on from the day after. Every row in + * the file is one that will not change, so a nightly run only ever appends. */ // Several internals are exported for tests. They are not in the package's public // surface: `bin` points at this file and `src/index.ts` does not re-export it, so @@ -34,14 +45,20 @@ // alternative is driving every branch through argv, which is how the resolution // layer ended up with no tests at all in the Python SDK. import { + closeSync, createWriteStream, existsSync, mkdirSync, + openSync, readFileSync, + readSync, + renameSync, rmSync, statSync, writeFileSync, + writeSync, } from 'node:fs'; +import { StringDecoder } from 'node:string_decoder'; import { basename, join } from 'node:path'; import { createHash } from 'node:crypto'; import { parseArgs } from 'node:util'; @@ -65,8 +82,25 @@ export const EX_TEMPFAIL = 75; // duplicated, exit 0. const STATE_SUFFIX = '.backfill-state.json'; -/** Bumped when a field changes meaning. A foreign version is refused, not guessed. */ -const STATE_VERSION = 1; +/** + * Bumped when a field changes meaning. A foreign version is refused, not guessed, + * with one exception: version 1, below. + * + * 2: `end` is the newest FINAL day, and nothing after it is in the file. In + * version 1 it was `retrievableThrough`, usually today, so the newest rows of a + * finished file were partial days that no later run replaced. Version 2 also + * records `since`, the first day the file was asked to start from. + */ +export const STATE_VERSION = 2; + +/** + * How far back from a version-1 file's `end` its rows may be partial. The + * archive records days 2 to 3 behind live data, so an 8.3 run that ended on its + * `retrievableThrough` wrote two or three days that were not final. A week covers + * that with room to spare, and every day fetched again costs nothing more than + * the one request its page already needs. + */ +export const V1_UNSETTLED_DAYS = 7; const SDK_NAME = 'js'; /** Where this park's state lives, for this format. */ @@ -125,6 +159,19 @@ function nextPageFrom(next: string): string | null { } } +/** + * True for a real calendar day written YYYY-MM-DD, the form the API itself uses. + * + * Strict on purpose, and the same rule as the Python SDK: `2025-1-1`, + * `20250101` and `2025-02-30` are all refused rather than read as something + * the person may not have meant. `Date` would roll the last one over to March. + */ +export function isoDay(value: string): boolean { + if (!/^\d{4}-\d{2}-\d{2}$/u.test(value)) return false; + const parsed = new Date(`${value}T00:00:00Z`); + return !Number.isNaN(parsed.getTime()) && parsed.toISOString().slice(0, 10) === value; +} + /** A park's identity, so rows can name themselves. */ interface Park { id: string; @@ -490,6 +537,14 @@ interface BackfillState { */ resumeFrom: string | null; complete: boolean; + /** + * The first day the file was ASKED to start from: `--since`, or where the + * archive starts, whichever is later. Not the same as `start` on a plan + * short of the full archive, where the request is clamped up to the first + * day the key may read. It is what lets a nightly cron with a fixed + * `--since` before that day keep running. Absent from version-1 files. + */ + since?: string | null; } function readState(path: string): BackfillState | null { @@ -514,12 +569,14 @@ function readState(path: string): BackfillState | null { * them duplicates, marked complete. */ function stateMismatch(state: BackfillState, format: string): string | null { - if (state.stateVersion !== STATE_VERSION) { - return `it was written by a different version of this command (state v${String(state.stateVersion)})`; - } + // The SDK first, so an old file from the other SDK says which SDK wrote it + // rather than blaming the version, which is not what needs fixing. if (state.sdk !== SDK_NAME) { return `it was written by the ${String(state.sdk)} SDK, and resuming across SDKs is not supported`; } + if (state.stateVersion !== STATE_VERSION) { + return `it was written by a different version of this command (state v${String(state.stateVersion)})`; + } if (state.format !== format) return `it is a ${String(state.format)} run`; if (state.columns !== columnsFingerprint(format)) { return 'the column layout changed since it was written'; @@ -539,6 +596,10 @@ interface RunOptions { outDir: string; format: 'ndjson' | 'csv'; overwrite: boolean; + /** First day to download, inclusive. Absent: as far back as the plan reaches. */ + since?: string | null; + /** Last day to download, inclusive. Absent: the newest final day. */ + until?: string | null; } /** A plan to proceed with, or an exit code meaning do not. */ @@ -553,31 +614,127 @@ type Decision = * nothing", which on a resumed run is not "the file is empty". */ resumed: boolean; + /** True when a FINISHED file is being carried forward to new final days. */ + extending: boolean; + /** The first day the file was asked to start from. See {@link BackfillState}. */ + since: string | null; + /** The newest day already in the file, kept if this run writes nothing. */ + priorLastDay: string | null; } | number; +/** YYYY-MM-DD for the day after `day`. */ +function nextDay(day: string): string { + const d = new Date(`${day}T00:00:00Z`); + d.setUTCDate(d.getUTCDate() + 1); + return d.toISOString().slice(0, 10); +} + +/** YYYY-MM-DD for `n` days before `day`. */ +function daysBefore(day: string, n: number): string { + const d = new Date(`${day}T00:00:00Z`); + d.setUTCDate(d.getUTCDate() - n); + return d.toISOString().slice(0, 10); +} + +/** The later of two days, either of which may be null. ISO days sort as text. */ +function later(a: string | null, b: string | null): string | null { + if (a === null) return b; + if (b === null) return a; + return a >= b ? a : b; +} + +function refuse(outPath: string, why: string): number { + process.stderr.write( + ` ${basename(outPath)}: ${why}.\n` + + ` --overwrite replace it with the range asked for\n` + + ` or pass a different --out and run again\n`, + ); + return 1; +} + +/** + * Null when `--since`/`--until` agree with the file being continued, else an + * exit code. + * + * A file here is one contiguous range of days, and a run can only append to it, + * so two requests cannot be honoured without rewriting it: a `--since` before + * the file starts, and one after the day it would continue from, which would + * leave a gap the state file could not describe. Both are refused rather than + * quietly ignored, which would hand back a file that is not what was asked for. + * + * A `--since` inside the file is fine and common: a cron line with a fixed + * `--since` runs it every night, and one computed as "30 days ago" moves + * forward every night. "Before the file" is judged against what the file was + * ASKED to start from, not where it does start: on a plan short of the full + * archive the first request is clamped up to the key's first day, and a fixed + * `--since` before that day has to keep working the next night. + */ +function rangeFits( + outPath: string, + state: BackfillState, + opts: RunOptions, + archiveFrom: string | null, + continueAt: string, +): number | null { + const since = opts.since != null ? later(opts.since, archiveFrom) : null; + const until = opts.until ?? null; + const fileStart = state.start; + const askedFrom = state.since ?? fileStart; + if (since !== null && askedFrom !== null && since < askedFrom) { + return refuse( + outPath, + `it was started from ${String(fileStart)}, and --since ${String(opts.since)} would ` + + `need days before that. Appending cannot add them`, + ); + } + if (until !== null && fileStart !== null && until < fileStart) { + return refuse(outPath, `it was started from ${fileStart}, after --until ${until}`); + } + if (since !== null && since > continueAt) { + return refuse( + outPath, + `it continues from ${continueAt}, so starting at --since ${String(opts.since)} ` + + `would leave a gap`, + ); + } + return null; +} + export function decide( outPath: string, statePath: string, opts: RunOptions, archiveFrom: string | null, + end: string | null, ): Decision { + if (end === null) { + // Nothing is final: a park the archive has not recorded a day of yet. + // Nothing is touched, so an existing file and its state stay as they are. + process.stderr.write( + ` nothing final to fetch into ${basename(outPath)} yet. The archive records ` + + `days 2 to 3 behind live data; run again later\n`, + ); + return 0; + } if (opts.overwrite) { rmSync(outPath, { force: true }); rmSync(statePath, { force: true }); } - const state = opts.overwrite ? null : readState(statePath); - const fileHasRows = existsSync(outPath) && statSync(outPath).size > 0; - const mismatch = state === null ? null : stateMismatch(state, opts.format); - const resumable = state !== null && mismatch === null; - - if (state?.complete === true && resumable && fileHasRows) { - process.stderr.write( - ` already complete: ${String(state.start)} .. ${String(state.end)} ` + - `in ${basename(outPath)} — pass --overwrite to fetch it again\n`, - ); - return 0; + let state = opts.overwrite ? null : readState(statePath); + const hasRows = (): boolean => existsSync(outPath) && statSync(outPath).size > 0; + let fileHasRows = hasRows(); + + // A state file describing a data file that is no longer there. Continuing + // would write a file that starts part-way through its range and then record + // it as complete. There is nothing to continue, so start again. + if (state !== null && !fileHasRows) state = null; + + if (state !== null && upgradable(state, opts.format)) { + state = upgradeV1(state, outPath, statePath, opts.format); + fileHasRows = hasRows(); } + if (fileHasRows && state === null) { // Appending would double it; truncating would destroy someone's data. process.stderr.write( @@ -587,31 +744,278 @@ export function decide( ); return 1; } - // A state file this build cannot resume. Refusing is the only safe answer: the - // file beside it was written to a different contract, and appending to it + // A state file this build cannot continue. Refusing is the only safe answer: + // the file beside it was written to a different contract, and appending to it // produces a file no reader can parse, or one that parses wrongly. - if (state !== null && mismatch !== null && state.complete !== true) { + // + // FINISHED OR NOT. Only an unfinished one used to be refused: a finished one + // fell through to a fresh start, and a fresh start opens the existing file in + // append mode, so the whole archive went in a second time under a second + // header, exit 0. A finished file is continued now, so it is refused too. + const mismatch = state === null ? null : stateMismatch(state, opts.format); + if (state !== null && mismatch !== null) { + const kind = state.complete ? 'a finished' : 'an unfinished'; process.stderr.write( - ` there is an unfinished ${basename(outPath)} beside this state file, but ${mismatch}.\n` + + ` there is ${kind} ${basename(outPath)} beside this state file, but ${mismatch}.\n` + ` --overwrite start this park again from the beginning\n` + ` or move both files aside and run again\n`, ); return 1; } - const resuming = resumable && state.complete !== true; - // `lastDay` is the fallback for the two cases with no page boundary to use: a - // state file written by an older version, and a run that died part-way through - // its FIRST page. It re-fetches one day, so that day's rows appear twice -- - // bad, and still far better than starting from the top and appending a second - // copy of everything, which is what an unconditional archiveFrom would do. + if (state === null) return firstRun(outPath, opts, archiveFrom, end); + return continueFile(outPath, state, opts, archiveFrom, end); +} + +/** A park with no file yet: from `--since`, or wherever the archive starts. */ +function firstRun( + outPath: string, + opts: RunOptions, + archiveFrom: string | null, + end: string, +): Decision { + const since = opts.since ?? null; + const start = since !== null ? later(since, archiveFrom) : archiveFrom; + if (since !== null && start !== null && start > end) { + process.stderr.write( + ` nothing to fetch into ${basename(outPath)}: --since ${since} is after ` + + `${end}, the newest final day\n`, + ); + return 0; + } return { - start: (resuming ? (state.resumeFrom ?? state.lastDay) : null) ?? archiveFrom, - hasRows: fileHasRows && resuming, - priorStart: resuming ? state.start : null, - resumed: resuming, + start, + hasRows: false, + priorStart: null, + resumed: false, + extending: false, + since: start, + priorLastDay: null, }; } +/** A file this command wrote: carry it forward, or say why not. */ +function continueFile( + outPath: string, + state: BackfillState, + opts: RunOptions, + archiveFrom: string | null, + end: string, +): Decision { + const carried = { + hasRows: true, + priorStart: state.start, + resumed: true, + since: state.since ?? state.start, + priorLastDay: state.lastDay, + }; + if (state.complete) { + // FINISHED IS NOT FOREVER. It used to be: a rerun printed "already + // complete" and exited 0 without asking for a single new day, so a nightly + // cron looked healthy and never updated. The file holds every final day + // through `end`, so the next day is where it carries on. + const continueAt = state.end !== null ? nextDay(state.end) : String(state.start); + const refused = rangeFits(outPath, state, opts, archiveFrom, continueAt); + if (refused !== null) return refused; + if (continueAt > end) { + process.stderr.write( + ` up to date: ${basename(outPath)} is complete through ${String(state.end)}, ` + + `and there is no final day after it yet\n`, + ); + return 0; + } + return { ...carried, start: continueAt, extending: true }; + } + // THE PAGE BOUNDARY, not the newest row. `lastDay` is the fallback for the one + // case with no boundary recorded: a run that died part-way through its FIRST + // page. It re-fetches one day, so that day's rows appear twice -- bad, and + // still far better than starting from the top and appending a second copy of + // everything. + const continueAt = state.resumeFrom ?? state.lastDay ?? state.start ?? archiveFrom; + const refused = rangeFits(outPath, state, opts, archiveFrom, String(continueAt)); + if (refused !== null) return refused; + return { ...carried, start: continueAt, extending: false }; +} + +// --------------------------------------------------------------------------- +// Files written by 8.3.x, whose newest rows may be partial days. +// --------------------------------------------------------------------------- + +/** A version-1 state file from this SDK, for this format and column layout. */ +function upgradable(state: BackfillState, format: string): boolean { + return ( + state.stateVersion === 1 && + state.sdk === SDK_NAME && + state.format === format && + state.columns === columnsFingerprint(format) + ); +} + +/** + * Make an 8.3 file one this build can continue, replacing its unsettled tail. + * + * 8.3 ended every run at `retrievableThrough`, usually today, so the last few + * days of a finished 8.3 file were written while they were still changing. + * Nothing ever replaced them, because a finished park was never fetched again. + * + * Which of those days were final at the time was not recorded, so every row + * dated within {@link V1_UNSETTLED_DAYS} of that run's end is removed and the + * state is set to carry on from the day after the cut. The next request then + * fetches those days again, final this time. A file whose newest row is already + * older than the cut, a park that stopped reporting long ago, is not read at all. + * + * A file that lies wholly inside the cut, as every anonymous 7-day file does, is + * started again instead: trimming would leave it empty with a state saying it + * continues from a day before the key can read. + * + * The new state is written straight away, so a run that fails after this point + * does not trim the same file twice. + */ +function upgradeV1( + state: BackfillState, + outPath: string, + statePath: string, + format: string, +): BackfillState | null { + const upgraded: BackfillState = { ...state, stateVersion: STATE_VERSION }; + if (state.end === null) return upgraded; + const keepThrough = daysBefore(state.end, V1_UNSETTLED_DAYS); + if (state.start !== null && keepThrough < state.start) { + rmSync(outPath, { force: true }); + rmSync(statePath, { force: true }); + return null; + } + const lastDay = state.lastDay; + if (lastDay === null || lastDay > keepThrough) { + trimAfter(outPath, format, keepThrough); + upgraded.lastDay = lastDay !== null ? keepThrough : null; + } + if (state.complete) { + upgraded.end = keepThrough; + } else { + const resume = state.resumeFrom ?? lastDay; + if (resume === null || resume > nextDay(keepThrough)) { + upgraded.resumeFrom = nextDay(keepThrough); + } + } + writeState(statePath, upgraded); + return upgraded; +} + +/** + * The cells of one CSV record, as written by {@link csvLine}: quoted where + * needed, `""` for a quote inside quotes. The record has no trailing newline. + */ +function csvCells(record: string): string[] { + const cells: string[] = []; + let cell = ''; + let quoted = false; + for (let i = 0; i < record.length; i += 1) { + const c = record[i]!; + if (quoted) { + if (c === '"' && record[i + 1] === '"') { + cell += '"'; + i += 1; + } else if (c === '"') { + quoted = false; + } else { + cell += c; + } + } else if (c === '"') { + quoted = true; + } else if (c === ',') { + cells.push(cell); + cell = ''; + } else { + cell += c; + } + } + cells.push(cell); + return cells; +} + +/** + * Remove every row dated after `keepThrough`, leaving the rest byte for byte. + * + * Streamed into a file beside the original and swapped in with one rename, so an + * interruption leaves either the old file or the new one, never half of each. + * A line or record that cannot be read is kept: this command does not get to + * decide that something it does not understand is worthless. + * + * Kept records are copied as the raw text they were read from, never parsed and + * written back, so quoting is untouched by construction. A CSV record ends at a + * newline outside quotes; `csvLine` quotes every cell holding a CR or LF, so a + * name with a line break in it stays one record. + */ +export function trimAfter(outPath: string, format: string, keepThrough: string): void { + const scratch = `${outPath}.trimming`; + const input = openSync(outPath, 'r'); + const output = openSync(scratch, 'w'); + try { + const decoder = new StringDecoder('utf8'); + const chunk = Buffer.alloc(1 << 20); + let pending = ''; + let quoted = false; + let scanned = 0; + let dateColumn: number | null = null; // null until the CSV header is read + let header = true; + + const keep = (record: string): boolean => { + const body = record.endsWith('\n') ? record.slice(0, -1) : record; + if (format !== 'csv') { + let day: unknown; + try { + day = (JSON.parse(body) as { date?: unknown } | null)?.date; + } catch { + day = undefined; + } + return !(typeof day === 'string' && day > keepThrough); + } + if (header) { + header = false; + const names = csvCells(body).map((name) => name.replace(/^\ufeff/u, '')); + dateColumn = names.indexOf('date'); + return true; + } + // No `date` column: not a file this can read, so it is copied whole. + if (dateColumn === null || dateColumn < 0) return true; + const day = csvCells(body)[dateColumn]; + return !(day !== undefined && day > keepThrough); + }; + + const drain = (final: boolean): void => { + let from = 0; + for (let i = scanned; i < pending.length; i += 1) { + const c = pending[i]; + if (format === 'csv' && c === '"') quoted = !quoted; + else if (c === '\n' && !(format === 'csv' && quoted)) { + const record = pending.slice(from, i + 1); + if (keep(record)) writeSync(output, record); + from = i + 1; + } + } + pending = pending.slice(from); + scanned = pending.length; + if (final && pending !== '') { + if (keep(pending)) writeSync(output, pending); + pending = ''; + } + }; + + for (;;) { + const read = readSync(input, chunk, 0, chunk.length, null); + if (read === 0) break; + pending += decoder.write(chunk.subarray(0, read)); + drain(false); + } + pending += decoder.end(); + drain(true); + } finally { + closeSync(input); + closeSync(output); + } + renameSync(scratch, outPath); +} + /** The minimum of a writable stream this module needs, so a test can stand in. */ interface Closable { end(cb: (error?: Error | null) => void): unknown; @@ -641,6 +1045,21 @@ export function flushAndClose(handle: Closable, pending: () => Error | null): Pr }); } +/** + * The last day this run asks for: the newest FINAL day, or `--until` if earlier. + * + * Not `retrievableThrough`. That is usually today, and today's row is the day so + * far; the archive records days 2 to 3 behind, so the days in between can still + * change too. Ending there wrote partial rows and, since a finished park was + * never fetched again, they stayed partial. + */ +function runEnd(span: HistorySpan, opts: RunOptions): string | null { + const final = span.finalThrough; + const until = opts.until ?? null; + if (final !== null && until !== null && until < final) return until; + return final; +} + export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions): Promise { const history = tp.entity(park.id).history; @@ -667,17 +1086,22 @@ export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions) const ext = opts.format === 'csv' ? 'csv' : 'ndjson'; const outPath = join(opts.outDir, `${park.id}.${ext}`); const statePath = statePathFor(opts.outDir, park.id, opts.format); - const end = span.retrievableThrough; + const end = runEnd(span, opts); - const decided = decide(outPath, statePath, opts, span.archiveFrom); + const decided = decide(outPath, statePath, opts, span.archiveFrom, end); if (typeof decided === 'number') return decided; let { start } = decided; - const { hasRows, priorStart, resumed } = decided; + const { hasRows, priorStart, resumed, extending, since, priorLastDay } = decided; - process.stderr.write( - `${park.id}: ${String(start)} .. ${String(end)}` + - `${priorStart !== null ? ' (resumed)' : ''} -> ${outPath}\n`, - ); + const note = extending ? ' (new days)' : resumed ? ' (resumed)' : ''; + process.stderr.write(`${park.id}: ${String(start)} .. ${String(end)}${note} -> ${outPath}\n`); + const through = span.retrievableThrough; + if (through !== null && end !== null && through > end && end === span.finalThrough) { + process.stderr.write( + ` stopping at ${end}: the days after it are still being recorded and can ` + + `change. The next run adds them once they are final\n`, + ); + } if (isEmptyWindow(start, end)) { process.stderr.write( @@ -726,8 +1150,30 @@ export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions) let lastDay: string | null = null; let resumeFrom: string | null = null; let skipped = false; + let outOfReach = false; + // The day this run's request started on, which is where a rerun has to carry + // on if the run fails before its first page is finished. + let firstDay: string | null = null; + + /** + * Where a rerun should continue, for the state file. + * + * The next page's start once a page is done. Before that, with rows written, + * null: `lastDay` is the fallback and costs one duplicated day. Before ANY + * row, the day this run began on. That last case had nothing recorded, which + * was harmless while only a first run could reach it -- a rerun of a file + * with no rows starts from the top anyway -- and is not now that a finished + * file is carried forward. A rerun interrupted before its first page would + * otherwise fall back to the file's start and append the whole range again. + */ + const resumePoint = (): string | null => { + if (resumeFrom !== null) return resumeFrom; + if (lastDay !== null) return null; + return firstDay; + }; const stream = async (from: string | null): Promise => { + firstDay = from; // `exactOptionalPropertyTypes` means an explicit undefined is not the same // as an absent key, so the query is built rather than spread with nulls. const query: { from?: string; to?: string; onPage: (page: HistoryPage) => void } = { @@ -777,9 +1223,10 @@ export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions) format: opts.format, start: priorStart ?? start, end, - lastDay, - resumeFrom, + lastDay: lastDay ?? priorLastDay, + resumeFrom: complete ? null : resumePoint(), complete, + since: since ?? priorStart ?? start, }); }; @@ -791,17 +1238,34 @@ export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions) // Retry only when nothing was written: a 403 mid-stream is not a plan // boundary, and restarting would duplicate rows. if (floor === null || written > 0) throw error; - process.stderr.write( - ` this key reaches back to ${floor}, not ${String(start)} — starting there\n`, - ); - start = floor; - if (isEmptyWindow(floor, end)) { + // A FILE BEING CONTINUED CANNOT JUMP FORWARD. Starting at the key's first + // day instead of the day the file continues from leaves a gap that the + // state file cannot describe, so the file would claim days it does not + // hold. It happens when a cron has not run for longer than the key's + // window, or the key lost its plan. Refused, with the file left alone. + if (resumed) { + if (start === null || floor <= start) throw error; process.stderr.write( - ` nothing in your window: this park's data ends ${String(end)} — skipping\n`, + ` this key reaches back to ${floor}, but ${basename(outPath)} continues from ` + + `${start}: the days between are out of reach, and carrying on from ${floor} ` + + `would leave a gap in the file. The rows already downloaded are left alone.\n` + + ` --overwrite start the file again from what this key can read\n` + + ` or pass a different --out and run again\n`, ); - skipped = true; + outOfReach = true; } else { - await stream(floor); + process.stderr.write( + ` this key reaches back to ${floor}, not ${String(start)} — starting there\n`, + ); + start = floor; + if (isEmptyWindow(floor, end)) { + process.stderr.write( + ` nothing in your window: this park's data ends ${String(end)} — skipping\n`, + ); + skipped = true; + } else { + await stream(floor); + } } } } catch (error) { @@ -833,6 +1297,9 @@ export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions) } await finish(); + // Nothing was written and the state is not touched, so the next run meets the + // same refusal until someone decides, rather than carrying on with a gap. + if (outOfReach) return 1; if (skipped && written === 0) { if (resumed) { // Same rule: keep the file, record where it got to, and say it did not finish. @@ -861,7 +1328,13 @@ options: --api-key KEY API key. Defaults to $THEMEPARKS_API_KEY. --format FORMAT ndjson (default) or csv --out DIR output directory (default: .) - --overwrite replace an existing file instead of refusing + --since DAY first day to download, YYYY-MM-DD, inclusive + (default: as far back as your plan reaches) + --until DAY last day to download, YYYY-MM-DD, inclusive + (default: the newest final day) + --overwrite replace an existing file instead of refusing. Without it, a + park that finished is brought up to date rather than fetched + twice, and a file this command did not write is never touched --help this --version print the package version @@ -882,6 +1355,23 @@ examples: themeparks-backfill 7340550b-c14d-4def-80bb-acdb51d49a66 --format csv --out ./data + themeparks-backfill "Epcot" --since 2025-01-01 + from a day of your choosing instead of as far back as your plan reaches. + --until YYYY-MM-DD sets the last day. Both are inclusive. + + themeparks-backfill "Epcot" (again, from cron, every night) + adds the days that became final since the last run, and nothing else. + +only final days are written. Today's row is the day so far, and the archive +records days 2 to 3 behind live data, so the newest days can still change. A run +ends at the newest final day and the next run carries on from the day after, so +the file only ever grows and no row in it changes later. + +--since applies when a file is started. A later run continues that file forward +and accepts the same --since, or a later one. One earlier than the file's first +day, or past the day it continues from, is refused: use --overwrite, or a +different --out. + exit codes: 0 done 75 the hourly history budget ran out. Progress is recorded; run the same @@ -924,6 +1414,8 @@ export async function main( format: { type: 'string', default: 'ndjson' }, out: { type: 'string', default: '.' }, overwrite: { type: 'boolean', default: false }, + since: { type: 'string' }, + until: { type: 'string' }, help: { type: 'boolean', default: false }, version: { type: 'boolean', default: false }, }, @@ -948,6 +1440,19 @@ export async function main( process.stderr.write(`--format must be ndjson or csv, not "${String(values.format)}"\n`); return 2; } + // CHECKED BEFORE ANY REQUEST, like every other argument error: a range that + // can never be satisfied should not spend a history request finding out. + for (const flag of ['since', 'until'] as const) { + const value = values[flag]; + if (value !== undefined && !isoDay(value)) { + process.stderr.write(`--${flag}: expected a day as YYYY-MM-DD, got "${value}"\n`); + return 2; + } + } + if (values.since !== undefined && values.until !== undefined && values.since > values.until) { + process.stderr.write(`--since ${values.since} is after --until ${values.until}\n`); + return 2; + } // `?? ` only catches null and undefined, so `THEMEPARKS_API_KEY=""` -- a // clobbered env var in a cron file, which is how this actually happens -- read @@ -1042,6 +1547,8 @@ export async function main( outDir: values.out, format: values.format, overwrite: values.overwrite, + ...(values.since !== undefined ? { since: values.since } : {}), + ...(values.until !== undefined ? { until: values.until } : {}), }; // ONE PARK'S FAILURE IS NOT THE DESTINATION'S. A 500 on Animal Kingdom used // to throw straight out of here, so the four parks after it were never diff --git a/test/mutation/mutants.json b/test/mutation/mutants.json index 5a40a06..0a08532 100644 --- a/test/mutation/mutants.json +++ b/test/mutation/mutants.json @@ -154,6 +154,132 @@ "find": " 'inParkHoursScheduledMinutes',", "replace": " 'inParkScheduledMinutes',", "why": "This SDK wrote 32 columns while Python wrote 41. The shared csv_contract.json fixture is what stops the two diverging again." + }, + { + "name": "a finished park is never fetched again", + "file": "src/backfill.ts", + "find": " if (continueAt > end) {", + "replace": " if (true) {", + "why": "A rerun printed 'already complete' and exited 0 without asking for a new day, so a nightly cron looked healthy and never updated." + }, + { + "name": "the run ends at retrievableThrough", + "file": "src/backfill.ts", + "find": "const final = span.finalThrough;", + "replace": "const final = span.retrievableThrough;", + "why": "retrievableThrough is usually today. Ending there wrote the day so far, and the unsettled days before it, as if they were final." + }, + { + "name": "--since is ignored on a first run", + "file": "src/backfill.ts", + "find": "const since = opts.since ?? null;", + "replace": "const since = null;", + "why": "The flag parsed but the download still started at the top of the archive." + }, + { + "name": "an earlier --since than the file is accepted", + "file": "src/backfill.ts", + "find": "if (since !== null && askedFrom !== null && since < askedFrom) {", + "replace": "if (false) {", + "why": "Appending cannot add days before the file's first one, so accepting it hands back a file that is not what was asked for." + }, + { + "name": "a --since past the file's end leaves a gap", + "file": "src/backfill.ts", + "find": "if (since !== null && since > continueAt) {", + "replace": "if (false) {", + "why": "Starting after the day the file continues from leaves a gap the state file cannot describe." + }, + { + "name": "a fixed --since before the key's floor is refused the next night", + "file": "src/backfill.ts", + "find": "const askedFrom = state.since ?? fileStart;", + "replace": "const askedFrom = fileStart;", + "why": "On a plan short of the archive the file starts at the key's first day, so judging --since against that refused the same cron line the next night." + }, + { + "name": "--since after --until is accepted", + "file": "src/backfill.ts", + "find": "values.since > values.until) {", + "replace": "values.since < '') {", + "why": "A range that can never be satisfied should be refused before any request is spent on it." + }, + { + "name": "a day that does not exist is accepted", + "file": "src/backfill.ts", + "find": "toISOString().slice(0, 10) === value", + "replace": "toISOString().length > 0", + "why": "Date rolls 2025-02-30 over to March 2; the person did not ask for March." + }, + { + "name": "a finished file with a foreign layout is appended to", + "file": "src/backfill.ts", + "find": "if (state !== null && mismatch !== null) {", + "replace": "if (state !== null && mismatch !== null && !state.complete) {", + "why": "Only an unfinished mismatched state was refused; a finished one got the whole archive appended under a second header, exit 0." + }, + { + "name": "a state without its data file is continued", + "file": "src/backfill.ts", + "find": "if (state !== null && !fileHasRows) state = null;", + "replace": "", + "why": "Continuing writes a file that starts part-way through its range and records it as complete." + }, + { + "name": "a rerun interrupted before its first page records no resume point", + "file": "src/backfill.ts", + "find": "return firstDay;", + "replace": "return null;", + "why": "The next run fell back to the file's start and appended the whole range again." + }, + { + "name": "an 8.3 file keeps its partial tail", + "file": "src/backfill.ts", + "find": "upgradable(state, opts.format)) {", + "replace": "upgradable(state, opts.format) && false) {", + "why": "8.3 wrote the newest days while they were still changing and never replaced them." + }, + { + "name": "the trim ends a CSV record at a quoted newline", + "file": "src/backfill.ts", + "find": "quoted = !quoted;", + "replace": "quoted = false;", + "why": "A name with a line break in it is one record, not two; splitting it corrupts the kept rows." + }, + { + "name": "an anonymous 8.3 file is trimmed to nothing", + "file": "src/backfill.ts", + "find": "keepThrough < state.start) {", + "replace": "keepThrough < '') {", + "why": "A file wholly inside the cut would be emptied and continued from a day the key can no longer read." + }, + { + "name": "a continued file jumps forward to the key's floor", + "file": "src/backfill.ts", + "find": " if (resumed) {", + "replace": " if (false) {", + "why": "Carrying on from the key's first day instead of the file's next day leaves a gap the file then claims not to have." + }, + { + "name": "changeRows drops opening", + "file": "src/ergonomic/history.ts", + "find": "return openingsOf(envelope);", + "replace": "return {};", + "why": "Without the opening, the time before an entity's first change has no known status: 63 seconds of Space Mountain on 2026-09-26." + }, + { + "name": "load() and iteration make two requests", + "file": "src/ergonomic/history.ts", + "find": "pending ??= ", + "replace": "pending = ", + "why": "Reading opening must cost the same single request the rows do." + }, + { + "name": "finalThrough is the later of the two days", + "file": "src/ergonomic/history.ts", + "find": "return a < b ? a : b;", + "replace": "return a > b ? a : b;", + "why": "The later day is retrievableThrough on most keys: today, whose row is not final." } ] } diff --git a/test/unit/backfill-incremental.test.ts b/test/unit/backfill-incremental.test.ts new file mode 100644 index 0000000..6144499 --- /dev/null +++ b/test/unit/backfill-incremental.test.ts @@ -0,0 +1,882 @@ +/** + * `themeparks-backfill` over time: a date range, reruns that add new days, and + * only final days in the file. + * + * Three defects from one customer-style run of 8.3.1, each reproduced against + * the live API before it was fixed: + * + * 1. No way to ask for less than everything. `--since` was "Unknown option", so + * a key that reaches the whole archive downloaded all of it, every time. + * 2. A finished park was finished forever. A rerun printed "already complete" + * and exited 0 without asking for a single new day, so a nightly cron looked + * healthy and never updated. The only way to get yesterday was + * `--overwrite`, which downloads the whole archive again. + * 3. The run ended at `retrievableThrough`, which is usually today. Today's row + * is the day so far, and the archive records days 2 to 3 behind, so the + * newest rows in the file were partial, and because of (2) they were never + * corrected. + * + * The stub below serves a park the way the API does: one row per entity per + * day, 31 days a page, final values through `recordedTo` and partial values + * after it. A partial row carries half the operating minutes, so a test can tell + * from the file alone whether a non-final day was ever written. The Python SDK + * tests the same behaviour with the same stub, case for case. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { + CSV_COLUMNS, + EX_TEMPFAIL, + STATE_VERSION, + backfillPark, + columnsFingerprint, + csvLine, + main, + ndjsonLine, + statePathFor, + trimAfter, +} from '../../src/backfill'; +import { ApiError } from '../../src/errors'; +import type { ThemeParks } from '../../src/client'; +import { + BudgetExhaustedError, + type DailyEntry, + type DaysOptions, + type HistoryApi, + type HistorySpan, +} from '../../src/ergonomic/history'; + +const FINAL_MINUTES = 600; +const PARTIAL_MINUTES = 300; +const PAGE_DAYS = 31; + +function addDays(day: string, n: number): string { + const d = new Date(`${day}T00:00:00Z`); + d.setUTCDate(d.getUTCDate() + n); + return d.toISOString().slice(0, 10); +} + +const minDay = (a: string, b: string) => (a < b ? a : b); + +function forbidden(earliest: string): ApiError { + return new ApiError(`403 Forbidden: This key can see history back to ${earliest}.`, { + status: 403, + body: { + error: { + type: 'HISTORY_WINDOW_EXCEEDED', + message: `This key can see history back to ${earliest} (400 days).`, + earliestAllowedDate: earliest, + }, + }, + url: 'https://api.themeparks.wiki/v1/entity/p/history/daily', + }); +} + +/** + * A park's daily history, served as the API serves it. + * + * `through` is retrievableThrough (usually today), `recordedTo` the newest day + * the archive holds. Days after `recordedTo` are served with partial values, + * which is what today's row and the days still being recorded look like. + */ +class Archive implements Pick { + calls: [string, string][] = []; + /** Throw BudgetExhaustedError on the Nth page request (1-based), or never. */ + budgetOnPage: number | null = null; + pagesServed = 0; + names: Record = {}; + + constructor( + public archiveFrom = '2026-06-01', + public recordedTo: string | null = '2026-09-26', + public through = '2026-09-28', + public floor: string | null = null, + public entities: string[] = ['ent-a', 'ent-b'], + ) {} + + /** Time passes: the archive and today both move on. */ + advance(days: number): void { + if (this.recordedTo !== null) this.recordedTo = addDays(this.recordedTo, days); + this.through = addDays(this.through, days); + } + + span(): Promise { + return Promise.resolve({ + archiveFrom: this.archiveFrom, + recordedTo: this.recordedTo, + retrievableThrough: this.through, + finalThrough: this.recordedTo === null ? null : minDay(this.recordedTo, this.through), + }); + } + + async *days(options: DaysOptions = {}): AsyncGenerator { + const from = String(options.from); + const to = String(options.to); + this.calls.push([from, to]); + if (this.floor !== null && from < this.floor) throw forbidden(this.floor); + const last = minDay(to, this.through); + let pageStart = from; + while (pageStart <= last) { + this.pagesServed += 1; + if (this.budgetOnPage === this.pagesServed) { + throw new BudgetExhaustedError('429', { + status: 429, + body: {}, + url: 'u', + retryAfterMs: 2_700_000, + }); + } + const pageEnd = minDay(addDays(pageStart, PAGE_DAYS - 1), last); + for (const entity of this.entities) { + for (let day = pageStart; day <= pageEnd; day = addDays(day, 1)) { + yield await Promise.resolve(this.entry(entity, day)); + } + } + const following = addDays(pageEnd, 1); + const next = + following <= last + ? `https://api.themeparks.wiki/v1/entity/p/history/daily?from=${following}&to=${last}` + : null; + options.onPage?.({ from: pageStart, to: pageEnd, next }); + pageStart = following; + } + } + + entry(entity: string, day: string): DailyEntry { + const final = this.recordedTo !== null && day <= this.recordedTo; + return { + entityId: entity, + name: this.names[entity] ?? `Ride ${entity}`, + entityType: 'ATTRACTION', + row: { + date: day, + firstOperatingAt: `${day}T13:00:00Z`, + lastClosedAt: final ? `${day}T23:00:00Z` : null, + operatingMinutes: final ? FINAL_MINUTES : PARTIAL_MINUTES, + downMinutes: 0, + changes: 3, + } as unknown as DailyEntry['row'], + }; + } +} + +const PARK = { id: 'p', name: 'Park', destination: '' }; + +let dir: string; +let err: string[]; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'bf-incr-')); + err = []; + vi.spyOn(process.stderr, 'write').mockImplementation((chunk) => { + err.push(String(chunk)); + return true; + }); + vi.spyOn(process.stdout, 'write').mockImplementation(() => true); +}); +afterEach(() => { + vi.restoreAllMocks(); + rmSync(dir, { recursive: true, force: true }); +}); + +const stderr = (): string => err.join(''); + +function run( + archive: Archive, + format: 'ndjson' | 'csv' = 'ndjson', + window: { since?: string; until?: string } = {}, + outDir = dir, +): Promise { + const tp = { entity: () => ({ history: archive }) } as unknown as ThemeParks; + return backfillPark(tp, PARK, { outDir, format, overwrite: false, ...window }); +} + +const dataPath = (format = 'ndjson', outDir = dir) => join(outDir, `p.${format}`); + +interface OutRow { + entityId: string; + date: string; + operatingMinutes: number; +} + +/** The rows in the file. The CSV here has no quoted cells, so a split is exact. */ +function rows(format: 'ndjson' | 'csv' = 'ndjson'): OutRow[] { + const text = readFileSync(dataPath(format), 'utf8'); + if (format === 'ndjson') { + return text + .split('\n') + .filter((l) => l !== '') + .map((l) => JSON.parse(l) as OutRow); + } + const [header, ...lines] = text.replace(/^/u, '').split('\n'); + const names = header!.split(','); + return lines + .filter((l) => l !== '') + .map((l) => { + const cells = l.split(','); + const at = (n: string) => cells[names.indexOf(n)] ?? ''; + return { + entityId: at('entityId'), + date: at('date'), + operatingMinutes: Number(at('operatingMinutes')), + }; + }); +} + +function state(format = 'ndjson'): Record { + return JSON.parse(readFileSync(statePathFor(dir, 'p', format), 'utf8')) as Record< + string, + unknown + >; +} + +function expectEveryRowFinalAndUnique(written: OutRow[]): void { + const seen = new Map(); + for (const r of written) { + const key = `${r.entityId}|${r.date}`; + seen.set(key, (seen.get(key) ?? 0) + 1); + } + const dupes = [...seen].filter(([, n]) => n > 1).map(([k]) => k); + expect(dupes, '(entityId, date) written more than once').toEqual([]); + const partial = written.filter((r) => r.operatingMinutes !== FINAL_MINUTES).map((r) => r.date); + expect([...new Set(partial)], 'non-final days reached the file').toEqual([]); +} + +const maxDate = (written: OutRow[]) => + written + .map((r) => r.date) + .sort() + .at(-1); +const minDate = (written: OutRow[]) => written.map((r) => r.date).sort()[0]; + +describe('the stub is the API', () => { + it('serves partial days past recordedTo', async () => { + // Guard the oracle: if the stub never served a partial row, "no partial row + // in the file" would pass whether or not the command stops early. + const archive = new Archive(); + const minutes: Record = {}; + for await (const e of archive.days({ from: '2026-09-25', to: '2026-09-28' })) { + minutes[(e.row as { date: string }).date] = ( + e.row as { operatingMinutes: number } + ).operatingMinutes; + } + expect(minutes['2026-09-26']).toBe(FINAL_MINUTES); + expect(minutes['2026-09-27']).toBe(PARTIAL_MINUTES); + expect(minutes['2026-09-28']).toBe(PARTIAL_MINUTES); + }); +}); + +describe('only final days are written', () => { + it('stops at the newest final day, and says why', async () => { + const archive = new Archive(); + expect(await run(archive)).toBe(0); + expect(archive.calls).toEqual([['2026-06-01', '2026-09-26']]); + expect(maxDate(rows())).toBe('2026-09-26'); + expectEveryRowFinalAndUnique(rows()); + expect(stderr()).toContain('stopping at 2026-09-26'); + expect(state().end).toBe('2026-09-26'); + expect(state().stateVersion).toBe(STATE_VERSION); + }); + + it('the next run adds those days once they are final', async () => { + // The other half: the days held back are not lost, they arrive on the next + // run with their final values, once. + const archive = new Archive(); + await run(archive); + archive.advance(2); + expect(await run(archive)).toBe(0); + expect(archive.calls.at(-1)).toEqual(['2026-09-27', '2026-09-28']); + expect(maxDate(rows())).toBe('2026-09-28'); + expectEveryRowFinalAndUnique(rows()); + }); + + it('writes nothing when nothing is final yet', async () => { + const archive = new Archive(); + archive.recordedTo = null; + expect(await run(archive)).toBe(0); + expect(archive.calls).toEqual([]); + expect(existsSync(dataPath())).toBe(false); + expect(stderr()).toContain('nothing final'); + }); + + it('says nothing about stopping when everything the park has is final', async () => { + // A park that stopped reporting: saying "stopping early" would be noise. + const archive = new Archive('2026-06-01', '2026-08-31', '2026-08-31'); + await run(archive); + expect(stderr()).not.toContain('stopping at'); + }); +}); + +describe('a rerun adds new days', () => { + it('fetches only the new final days, and leaves the rows already there alone', async () => { + const archive = new Archive(); + await run(archive); + const before = readFileSync(dataPath(), 'utf8'); + archive.advance(1); + expect(await run(archive)).toBe(0); + expect(archive.calls.at(-1)).toEqual(['2026-09-27', '2026-09-27']); + const after = readFileSync(dataPath(), 'utf8'); + expect(after.startsWith(before), 'the rows already there changed').toBe(true); + expect(rows()).toHaveLength(before.split('\n').length - 1 + 2); // two entities, one day + expectEveryRowFinalAndUnique(rows()); + expect(stderr()).toContain('(new days)'); + }); + + it('keeps the original start and moves the end', async () => { + const archive = new Archive(); + await run(archive); + archive.advance(3); + await run(archive); + expect(state()).toMatchObject({ start: '2026-06-01', end: '2026-09-29', complete: true }); + }); + + it('asks for nothing when there is nothing new', async () => { + const archive = new Archive(); + await run(archive); + const before = readFileSync(dataPath()); + expect(await run(archive)).toBe(0); + expect(archive.calls, 'asked the API again with nothing to ask for').toHaveLength(1); + expect(readFileSync(dataPath())).toEqual(before); + expect(stderr()).toContain('up to date'); + }); + + it('a nightly cron for a week leaves no gap and no overlap', async () => { + const archive = new Archive(); + for (let i = 0; i < 7; i += 1) { + expect(await run(archive)).toBe(0); + archive.advance(1); + } + expect(maxDate(rows())).toBe('2026-10-02'); + expectEveryRowFinalAndUnique(rows()); + const days = new Set(rows().map((r) => r.date)); + const expected = (Date.parse('2026-10-02') - Date.parse('2026-06-01')) / 86_400_000 + 1; + expect(days.size, 'a gap or an overlap between runs').toBe(expected); + }); + + it('a CSV rerun gains no second header or BOM', async () => { + const archive = new Archive(); + await run(archive, 'csv'); + archive.advance(2); + await run(archive, 'csv'); + const raw = readFileSync(dataPath('csv'), 'utf8'); + expect(raw.split('')).toHaveLength(2); + expect(raw.split('parkId,')).toHaveLength(2); + expectEveryRowFinalAndUnique(rows('csv')); + }); + + it('an interrupted rerun continues from its own start', async () => { + // The budget runs out on the rerun's FIRST request, before any page. With no + // page boundary and no row, the only thing saying where this run began is + // the state file; without it the next run started over from the top of the + // archive and appended a second copy of everything. + const archive = new Archive(); + await run(archive); + const before = rows().length; + archive.advance(40); + archive.budgetOnPage = archive.pagesServed + 1; + expect(await run(archive)).toBe(EX_TEMPFAIL); + expect(rows(), 'the file changed on a failed run').toHaveLength(before); + expect(state().resumeFrom).toBe('2026-09-27'); + + archive.budgetOnPage = null; + expect(await run(archive)).toBe(0); + expect(archive.calls.at(-1)?.[0]).toBe('2026-09-27'); + expectEveryRowFinalAndUnique(rows()); + }); + + it('an interrupted rerun part-way through resumes at the page boundary', async () => { + const archive = new Archive(); + await run(archive); + archive.advance(40); // two pages of new days + archive.budgetOnPage = archive.pagesServed + 2; + expect(await run(archive)).toBe(EX_TEMPFAIL); + expect(state().resumeFrom).toBe('2026-10-28'); + archive.budgetOnPage = null; + expect(await run(archive)).toBe(0); + expect(archive.calls.at(-1)?.[0]).toBe('2026-10-28'); + expectEveryRowFinalAndUnique(rows()); + }); + + it('refuses a rerun whose next day is older than the key reaches, and keeps the file', async () => { + // A cron that did not run for longer than the window, or a key that lost + // its plan. Carrying on from the key's first day would leave a gap the state + // file cannot describe, so the file would claim days it does not hold. + const archive = new Archive(); + await run(archive); + const before = readFileSync(dataPath()); + archive.advance(40); + archive.floor = '2026-10-10'; + expect(await run(archive)).toBe(1); + expect(readFileSync(dataPath())).toEqual(before); + expect(stderr()).toContain('gap'); + expect(stderr()).toContain('--overwrite'); + expect(state()).toMatchObject({ end: '2026-09-26', complete: true }); + }); +}); + +describe('--since and --until', () => { + it('--since is where the first request starts', async () => { + const archive = new Archive(); + expect(await run(archive, 'ndjson', { since: '2026-09-01' })).toBe(0); + expect(archive.calls).toEqual([['2026-09-01', '2026-09-26']]); + expect(minDate(rows())).toBe('2026-09-01'); + expect(state().start).toBe('2026-09-01'); + }); + + it('--until is where it ends, and is not reported as holding days back', async () => { + const archive = new Archive(); + await run(archive, 'ndjson', { since: '2026-07-01', until: '2026-07-31' }); + expect(archive.calls).toEqual([['2026-07-01', '2026-07-31']]); + expect(stderr()).not.toContain('stopping at'); + }); + + it('--until past the newest final day still stops at the final day', async () => { + const archive = new Archive(); + await run(archive, 'ndjson', { until: '2026-12-31' }); + expect(archive.calls).toEqual([['2026-06-01', '2026-09-26']]); + expectEveryRowFinalAndUnique(rows()); + }); + + it('--since before the archive starts at the archive', async () => { + const archive = new Archive(); + await run(archive, 'ndjson', { since: '2019-01-01' }); + expect(archive.calls[0]?.[0]).toBe('2026-06-01'); + }); + + it("--since before the plan's floor starts at the floor", async () => { + const archive = new Archive(); + archive.floor = '2026-08-01'; + expect(await run(archive, 'ndjson', { since: '2026-07-01' })).toBe(0); + expect(archive.calls.map((c) => c[0])).toEqual(['2026-07-01', '2026-08-01']); + expect(stderr()).toContain('reaches back to 2026-08-01'); + }); + + it('--since after the newest final day writes nothing', async () => { + const archive = new Archive(); + expect(await run(archive, 'ndjson', { since: '2026-09-27' })).toBe(0); + expect(archive.calls).toEqual([]); + expect(existsSync(dataPath())).toBe(false); + expect(stderr()).toContain('2026-09-26'); + }); + + it('the same --since on every run continues the file', async () => { + // A cron line with a fixed --since, run nightly. + const archive = new Archive(); + await run(archive, 'ndjson', { since: '2026-09-01' }); + archive.advance(1); + expect(await run(archive, 'ndjson', { since: '2026-09-01' })).toBe(0); + expect(archive.calls.at(-1)).toEqual(['2026-09-27', '2026-09-27']); + expectEveryRowFinalAndUnique(rows()); + }); + + it('the same --since before the archive is not a change', async () => { + const archive = new Archive(); + await run(archive, 'ndjson', { since: '2019-01-01' }); + archive.advance(1); + expect(await run(archive, 'ndjson', { since: '2019-01-01' })).toBe(0); + }); + + it('a rolling --since continues the file', async () => { + // `--since $(date -d '-30 days' +%F)` in a cron: later every night, and + // always inside the file, so the file just carries on. + const archive = new Archive(); + await run(archive, 'ndjson', { since: '2026-08-27' }); + archive.advance(1); + expect(await run(archive, 'ndjson', { since: '2026-08-28' })).toBe(0); + expect(archive.calls.at(-1)).toEqual(['2026-09-27', '2026-09-27']); + }); + + it('an earlier --since than the file is refused, with what to do', async () => { + // Appending older days after newer ones cannot make the file start earlier + // without rewriting it, and pretending otherwise records a range the file + // does not hold. + const archive = new Archive(); + await run(archive, 'ndjson', { since: '2026-09-01' }); + const before = readFileSync(dataPath()); + expect(await run(archive, 'ndjson', { since: '2026-08-01' })).toBe(1); + expect(readFileSync(dataPath())).toEqual(before); + expect(stderr()).toContain('2026-09-01'); + expect(stderr()).toContain('--overwrite'); + }); + + it('a --since that would leave a gap is refused', async () => { + const archive = new Archive(); + await run(archive, 'ndjson', { until: '2026-07-31' }); + expect(await run(archive, 'ndjson', { since: '2026-09-01' })).toBe(1); + expect(stderr()).toContain('gap'); + }); + + it('an --until before the file is refused', async () => { + const archive = new Archive(); + await run(archive, 'ndjson', { since: '2026-09-01' }); + expect(await run(archive, 'ndjson', { until: '2026-08-15' })).toBe(1); + expect(stderr()).toContain('--overwrite'); + }); + + it('--until, then no --until, extends the file', async () => { + const archive = new Archive(); + await run(archive, 'ndjson', { until: '2026-07-31' }); + expect(await run(archive)).toBe(0); + expect(archive.calls.at(-1)).toEqual(['2026-08-01', '2026-09-26']); + expectEveryRowFinalAndUnique(rows()); + }); + + it('an --until the file already covers is up to date', async () => { + const archive = new Archive(); + await run(archive); + expect(await run(archive, 'ndjson', { until: '2026-08-01' })).toBe(0); + expect(archive.calls).toHaveLength(1); + expect(stderr()).toContain('up to date'); + }); + + it("a fixed --since before a limited plan's floor keeps working every night", async () => { + // A free key reads 30 days, so `--since 2025-01-01` on a cron starts the + // file at the key's first day, not at 2025-01-01. The same line the next + // night asks for nothing the first run did not also ask for, so it must + // carry on, not be refused for predating the file's first row. + const archive = new Archive('2021-07-03'); + archive.floor = '2026-08-28'; + expect(await run(archive, 'ndjson', { since: '2025-01-01' })).toBe(0); + expect(state()).toMatchObject({ start: '2026-08-28', since: '2025-01-01' }); + archive.advance(1); + archive.floor = '2026-08-29'; + expect(await run(archive, 'ndjson', { since: '2025-01-01' })).toBe(0); + expect(archive.calls.at(-1)).toEqual(['2026-09-27', '2026-09-27']); + expectEveryRowFinalAndUnique(rows()); + }); + + it('a --since earlier than the one the file was started with is still refused', async () => { + const archive = new Archive('2021-07-03'); + archive.floor = '2026-08-28'; + await run(archive, 'ndjson', { since: '2025-01-01' }); + expect(await run(archive, 'ndjson', { since: '2024-01-01' })).toBe(1); + expect(stderr()).toContain('--overwrite'); + }); +}); + +// --------------------------------------------------------------------------- +// The command line, end to end through main() and a stubbed fetch. +// --------------------------------------------------------------------------- + +describe('the command line', () => { + const MK = '75ea578a-adc8-4116-a54d-dccb60765ef9'; + const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + + /** Answers the catalogue, coverage and an empty last page; records every URL. */ + function server() { + const seen: string[] = []; + const fetchFn = vi.fn((input: unknown) => { + const url = String(input); + seen.push(url); + if (url.includes('/destinations')) { + return Promise.resolve( + json({ + destinations: [ + { id: 'd', name: 'Dest', slug: 'd', parks: [{ id: MK, name: 'Magic Kingdom' }] }, + ], + }), + ); + } + if (url.includes('/history/coverage')) { + return Promise.resolve( + json({ + id: MK, + name: 'Magic Kingdom', + entityType: 'PARK', + summary: { + entitiesWithData: 1, + archiveFrom: '2021-07-03', + recordedTo: '2026-09-26', + retrievableThrough: '2026-09-28', + measuredOn: '2026-09-28', + }, + fields: {}, + entities: [], + }), + ); + } + const q = new URL(url).searchParams; + return Promise.resolve( + json({ + id: MK, + name: 'Magic Kingdom', + entityType: 'PARK', + timezone: 'America/New_York', + range: { from: q.get('from'), to: q.get('to') }, + entities: [], + next: null, + }), + ); + }); + return { fetchFn, seen }; + } + + const daily = (seen: string[]) => { + const url = seen.find((u) => u.includes('/history/daily')); + return url === undefined ? null : new URL(url).searchParams; + }; + + it('--since and --until reach the request', async () => { + const { fetchFn, seen } = server(); + const argv = [MK, '--since', '2025-01-01', '--until', '2025-12-31', '--out', dir]; + expect(await main([...argv, '--api-key', 'k'], { fetch: fetchFn })).toBe(0); + expect(daily(seen)?.get('from')).toBe('2025-01-01'); + expect(daily(seen)?.get('to')).toBe('2025-12-31'); + }); + + it('neither is required, and the run ends at the newest final day', async () => { + const { fetchFn, seen } = server(); + expect(await main([MK, '--out', dir, '--api-key', 'k'], { fetch: fetchFn })).toBe(0); + expect(daily(seen)?.get('from')).toBe('2021-07-03'); + expect(daily(seen)?.get('to')).toBe('2026-09-26'); + }); + + it.each(['2025-13-01', '2025-02-30', '2025-1-1', '20250101', 'yesterday', ''])( + 'refuses %j as a day, before any request', + async (bad) => { + const { fetchFn, seen } = server(); + expect( + await main([MK, '--since', bad, '--api-key', 'k', '--out', dir], { fetch: fetchFn }), + ).toBe(2); + expect( + await main([MK, '--until', bad, '--api-key', 'k', '--out', dir], { fetch: fetchFn }), + ).toBe(2); + expect(stderr()).toContain('YYYY-MM-DD'); + expect(seen).toEqual([]); + }, + ); + + it('refuses --since after --until, before any request', async () => { + const { fetchFn, seen } = server(); + const argv = [ + MK, + '--since', + '2025-06-01', + '--until', + '2025-05-31', + '--api-key', + 'k', + '--out', + dir, + ]; + expect(await main(argv, { fetch: fetchFn })).toBe(2); + expect(stderr()).toContain('--since 2025-06-01 is after --until 2025-05-31'); + expect(seen).toEqual([]); + }); + + it('--help documents both, and the final-day rule', async () => { + const out: string[] = []; + vi.mocked(process.stdout.write).mockImplementation((chunk) => { + out.push(String(chunk)); + return true; + }); + expect(await main(['--help'])).toBe(0); + const text = out.join(''); + expect(text).toContain('--since'); + expect(text).toContain('--until'); + expect(text).toContain('final'); + expect(text).toContain('again'); + }); +}); + +// --------------------------------------------------------------------------- +// Files written by 8.3.x: their newest days may be partial. +// --------------------------------------------------------------------------- + +/** A state file with the fields and values 8.3.1 writes for a finished park. */ +function v1State(format: 'ndjson' | 'csv', over: Record = {}): void { + writeFileSync( + statePathFor(dir, 'p', format), + `${JSON.stringify({ + sdk: 'js', + sdkVersion: '8.3.1', + stateVersion: 1, + columns: columnsFingerprint(format), + format, + start: '2026-06-01', + end: '2026-09-28', + lastDay: '2026-09-28', + resumeFrom: null, + complete: true, + ...over, + })}\n`, + ); +} + +/** The file 8.3.1 left behind: every day through `through`, partial tail included. */ +async function writeAs83(format: 'ndjson' | 'csv', archive: Archive, outDir = dir): Promise { + let text = format === 'csv' ? `${CSV_COLUMNS.join(',')}\n` : ''; + for await (const entry of archive.days({ from: archive.archiveFrom, to: archive.through })) { + text += format === 'csv' ? `${csvLine(PARK, entry)}\n` : ndjsonLine(PARK, entry); + } + writeFileSync(dataPath(format, outDir), text); + archive.calls = []; +} + +describe('a file 8.3 wrote is corrected once, not frozen', () => { + it.each(['ndjson', 'csv'] as const)( + 'replaces the partial tail with final rows (%s)', + async (format) => { + const archive = new Archive(); + await writeAs83(format, archive); + v1State(format); + const partialBefore = rows(format).filter((r) => r.operatingMinutes !== FINAL_MINUTES); + expect(partialBefore.length, 'the 8.3 file this starts from has no partial rows').toBe(4); + + expect(await run(archive, format)).toBe(0); + // Seven days back from the old end, so every day that could have been + // partial is fetched again, and nothing earlier is. + expect(archive.calls).toEqual([['2026-09-22', '2026-09-26']]); + expect(maxDate(rows(format))).toBe('2026-09-26'); + expect(minDate(rows(format))).toBe('2026-06-01'); + expectEveryRowFinalAndUnique(rows(format)); + expect(state(format).stateVersion).toBe(STATE_VERSION); + + archive.advance(2); + await run(archive, format); + expectEveryRowFinalAndUnique(rows(format)); + }, + ); + + it('keeps the rows it keeps byte for byte, awkward names included', async () => { + // Compared against a file written only up to the cut in the first place. + const nasty = { 'ent-a': 'Space, "Mountain"\rFastPass\n2', 'ent-b': "=cmd|' /C calc'!A0" }; + const archive = new Archive('2026-09-01'); + archive.names = nasty; + await writeAs83('csv', archive); + trimAfter(dataPath('csv'), 'csv', '2026-09-21'); + + const expectedDir = join(dir, 'expected'); + mkdirSync(expectedDir); + const short = new Archive('2026-09-01', '2026-09-21', '2026-09-21'); + short.names = nasty; + await writeAs83('csv', short, expectedDir); + expect(readFileSync(dataPath('csv'))).toEqual(readFileSync(dataPath('csv', expectedDir))); + }); + + it('keeps an NDJSON line it cannot parse', () => { + // Not this command's to judge: a line it cannot read is left where it is. + writeFileSync(dataPath(), '{"date":"2026-09-01"}\n{"date":"2026-09-27"}\n{"trunc\n'); + trimAfter(dataPath(), 'ndjson', '2026-09-21'); + expect(readFileSync(dataPath(), 'utf8')).toBe('{"date":"2026-09-01"}\n{"trunc\n'); + }); + + it('copies a CSV with no date column whole', () => { + writeFileSync(dataPath('csv'), 'a,b\n1,2099-01-01\n'); + trimAfter(dataPath('csv'), 'csv', '2026-09-21'); + expect(readFileSync(dataPath('csv'), 'utf8')).toBe('a,b\n1,2099-01-01\n'); + }); + + it('trims an interrupted 8.3 run that reached its last pages too', async () => { + const archive = new Archive(); + await writeAs83('ndjson', archive); + v1State('ndjson', { complete: false, lastDay: '2026-09-28', resumeFrom: null }); + expect(await run(archive)).toBe(0); + expect(archive.calls).toEqual([['2026-09-22', '2026-09-26']]); + expectEveryRowFinalAndUnique(rows()); + }); + + it('does not rewrite an old file whose newest day is long final', async () => { + // A park that stopped reporting: its 8.3 state ends months after its last + // row, and every row is final. Rewriting a large file to remove nothing is + // waste, so the file is not even opened for it. + const archive = new Archive('2026-06-01', '2026-07-31', '2026-07-31'); + await writeAs83('ndjson', archive); + v1State('ndjson', { end: '2026-09-28', lastDay: '2026-07-31' }); + const inode = statSync(dataPath()).ino; + expect(await run(archive)).toBe(0); + expect(statSync(dataPath()).ino).toBe(inode); + }); + + it('resumes an interrupted 8.3 run where it stopped, when it was far from the end', async () => { + const archive = new Archive(); + writeFileSync(dataPath(), '{"date":"2026-06-01"}\n'); + v1State('ndjson', { complete: false, lastDay: '2026-06-30', resumeFrom: '2026-07-01' }); + expect(await run(archive)).toBe(0); + expect(archive.calls).toEqual([['2026-07-01', '2026-09-26']]); + }); + + it('starts again an anonymous 8.3 file that lies wholly inside the cut', async () => { + // Anonymous access reads 7 days, so every row such a file holds is within + // seven days of its end. Trimming would leave it empty, continuing from a day + // the key can no longer read; it is fetched again instead, all final. + const anonymous = new Archive('2026-09-22'); + await writeAs83('ndjson', anonymous); + v1State('ndjson', { start: '2026-09-22' }); + // Two days later, the key's 7-day window has moved on past the file's start. + const archive = new Archive('2026-06-01', '2026-09-28', '2026-09-30', '2026-09-24'); + expect(await run(archive)).toBe(0); + expect(archive.calls.at(-1)).toEqual(['2026-09-24', '2026-09-28']); + expect(minDate(rows())).toBe('2026-09-24'); + expectEveryRowFinalAndUnique(rows()); + expect(state()).toMatchObject({ stateVersion: STATE_VERSION, end: '2026-09-28' }); + }); + + it('still refuses an old state file from the other SDK', async () => { + const archive = new Archive(); + writeFileSync(dataPath(), '{"date":"2026-06-01"}\n'); + v1State('ndjson', { sdk: 'py' }); + expect(await run(archive)).toBe(1); + expect(stderr()).toContain('py SDK'); + }); +}); + +describe('a finished file written to another contract is refused', () => { + it('does not append to a finished CSV with another column layout', async () => { + // Found while making reruns incremental: only an UNFINISHED mismatched state + // was refused. A finished one fell through to a fresh start, and the fresh + // start opened the existing file in append mode: a second full copy of the + // archive under a second header, exit 0. + writeFileSync(dataPath('csv'), 'old,header\n1,2\n'); + writeFileSync( + statePathFor(dir, 'p', 'csv'), + JSON.stringify({ + sdk: 'js', + sdkVersion: '9.9.9', + stateVersion: STATE_VERSION, + columns: '0000deadbeef0000', + format: 'csv', + start: '2026-06-01', + end: '2026-09-20', + lastDay: '2026-09-20', + resumeFrom: null, + complete: true, + }), + ); + const archive = new Archive(); + expect(await run(archive, 'csv')).toBe(1); + expect(archive.calls).toEqual([]); + expect(readFileSync(dataPath('csv'), 'utf8')).toBe('old,header\n1,2\n'); + expect(stderr()).toContain('column layout changed'); + }); +}); + +describe('a state file with no data file beside it', () => { + it('downloads the park again rather than continuing into a new file', async () => { + // The state describes a file that is no longer there. Continuing would write + // a file that starts part-way through and record it as complete. + const archive = new Archive(); + await run(archive); + rmSync(dataPath()); + archive.advance(1); + expect(await run(archive)).toBe(0); + expect(archive.calls.at(-1)?.[0]).toBe('2026-06-01'); + expect(minDate(rows())).toBe('2026-06-01'); + expectEveryRowFinalAndUnique(rows()); + }); +}); diff --git a/test/unit/backfill.test.ts b/test/unit/backfill.test.ts index 32b71bc..b41bf44 100644 --- a/test/unit/backfill.test.ts +++ b/test/unit/backfill.test.ts @@ -46,6 +46,7 @@ import { resolve as resolveParks, windowFloor, EX_TEMPFAIL, + STATE_VERSION, } from '../../src/backfill'; import { ApiError } from '../../src/errors'; import { PACKAGE_VERSION } from '../../src/client'; @@ -112,7 +113,7 @@ function stateFile( JSON.stringify({ sdk: 'js', sdkVersion: PACKAGE_VERSION, - stateVersion: 1, + stateVersion: STATE_VERSION, columns: columnsFingerprint(format), format, start: '2025-01-01', @@ -443,15 +444,18 @@ describe('decide', () => { }); it('starts at the archive floor when there is nothing there', () => { - expect(decide(out, state, opts(), '2021-07-03')).toEqual({ + expect(decide(out, state, opts(), '2021-07-03', '2026-09-23')).toEqual({ start: '2021-07-03', hasRows: false, priorStart: null, resumed: false, + extending: false, + since: '2021-07-03', + priorLastDay: null, }); }); - it('does nothing and exits 0 when the park is already complete', () => { + it('asks for nothing and exits 0 when a finished park has no new final day', () => { writeFileSync(out, '{"a":1}\n'); stateFile(dir, { start: '2021-07-03', @@ -460,19 +464,19 @@ describe('decide', () => { resumeFrom: null, complete: true, }); - expect(decide(out, state, opts(), '2021-07-03')).toBe(0); + expect(decide(out, state, opts(), '2021-07-03', '2026-09-23')).toBe(0); expect(readFileSync(out, 'utf8')).toBe('{"a":1}\n'); }); it('refuses a file with rows and no state beside it', () => { // Appending would double someone's data; truncating would destroy it. writeFileSync(out, '{"a":1}\n'); - expect(decide(out, state, opts(), '2021-07-03')).toBe(1); + expect(decide(out, state, opts(), '2021-07-03', '2026-09-23')).toBe(1); expect(readFileSync(out, 'utf8')).toBe('{"a":1}\n'); }); it('refuses to switch format part-way through a park', () => { - writeFileSync(out, '{"a":1}\n'); + writeFileSync(join(dir, 'p.csv'), 'a\n1\n'); stateFile(dir, { start: '2021-07-03', end: '2026-09-23', @@ -480,7 +484,9 @@ describe('decide', () => { resumeFrom: '2024-01-02', complete: false, }); - expect(decide(join(dir, 'p.csv'), state, opts({ format: 'csv' }), '2021-07-03')).toBe(1); + expect( + decide(join(dir, 'p.csv'), state, opts({ format: 'csv' }), '2021-07-03', '2026-09-23'), + ).toBe(1); }); it('resumes from the page boundary, not the newest row', () => { @@ -497,11 +503,12 @@ describe('decide', () => { resumeFrom: '2026-09-01', complete: false, }); - expect(decide(out, state, opts(), '2021-07-03')).toEqual({ + expect(decide(out, state, opts(), '2021-07-03', '2026-09-23')).toMatchObject({ start: '2026-09-01', hasRows: true, priorStart: '2021-07-03', resumed: true, + extending: false, }); }); @@ -515,7 +522,7 @@ describe('decide', () => { lastDay: '2026-08-30', complete: false, }); - expect(decide(out, state, opts(), '2021-07-03')).toMatchObject({ + expect(decide(out, state, opts(), '2021-07-03', '2026-09-23')).toMatchObject({ start: '2026-08-30', hasRows: true, }); @@ -524,18 +531,20 @@ describe('decide', () => { it('treats corrupt state as a file it must not touch', () => { writeFileSync(out, '{"a":1}\n'); writeFileSync(state, 'not json'); - expect(decide(out, state, opts(), '2021-07-03')).toBe(1); + expect(decide(out, state, opts(), '2021-07-03', '2026-09-23')).toBe(1); }); it('--overwrite clears both files first', () => { writeFileSync(out, '{"a":1}\n'); stateFile(dir, { complete: true }); - expect(decide(out, state, opts({ overwrite: true }), '2021-07-03')).toEqual({ - start: '2021-07-03', - hasRows: false, - priorStart: null, - resumed: false, - }); + expect(decide(out, state, opts({ overwrite: true }), '2021-07-03', '2026-09-23')).toMatchObject( + { + start: '2021-07-03', + hasRows: false, + priorStart: null, + resumed: false, + }, + ); expect(existsSync(out)).toBe(false); expect(existsSync(state)).toBe(false); }); @@ -757,7 +766,7 @@ describe('a whole run', () => { expect(await mainWith(second.fetchFn, [MK, '--out', dir])).toBe(0); expect(readFileSync(join(dir, `${MK}.ndjson`), 'utf8')).toBe(before); expect(second.calls()).toBe(0); - expect(err.join('')).toContain('already complete'); + expect(err.join('')).toContain('up to date'); }); it('a typo exits 1 with the listing hint and no traceback', async () => { @@ -1017,7 +1026,13 @@ describe('a state file this build cannot resume', () => { writeFileSync(join(dir, 'p.csv'), 'old,header\n1,2\n'); stateFile(dir, { columns: '0000deadbeef0000', lastDay: '2026-08-30' }, 'p', 'csv'); expect( - decide(join(dir, 'p.csv'), statePathFor(dir, 'p', 'csv'), opts('csv'), '2021-07-03'), + decide( + join(dir, 'p.csv'), + statePathFor(dir, 'p', 'csv'), + opts('csv'), + '2021-07-03', + '2026-09-23', + ), ).toBe(1); expect(err.join('')).toContain('column layout changed'); }); @@ -1029,7 +1044,13 @@ describe('a state file this build cannot resume', () => { writeFileSync(join(dir, 'p.ndjson'), '{"a":1}\n'); stateFile(dir, { sdk: 'py', lastDay: '2026-08-30' }); expect( - decide(join(dir, 'p.ndjson'), statePathFor(dir, 'p', 'ndjson'), opts(), '2021-07-03'), + decide( + join(dir, 'p.ndjson'), + statePathFor(dir, 'p', 'ndjson'), + opts(), + '2021-07-03', + '2026-09-23', + ), ).toBe(1); expect(err.join('')).toContain('py SDK'); }); @@ -1038,7 +1059,13 @@ describe('a state file this build cannot resume', () => { writeFileSync(join(dir, 'p.ndjson'), '{"a":1}\n'); stateFile(dir, { stateVersion: 99, lastDay: '2026-08-30' }); expect( - decide(join(dir, 'p.ndjson'), statePathFor(dir, 'p', 'ndjson'), opts(), '2021-07-03'), + decide( + join(dir, 'p.ndjson'), + statePathFor(dir, 'p', 'ndjson'), + opts(), + '2021-07-03', + '2026-09-23', + ), ).toBe(1); expect(err.join('')).toContain('different version'); }); From aeadf090f9f881f19d4fc999b18a849c885b6527 Mon Sep 17 00:00:00 2001 From: Jamie Holding Date: Mon, 28 Sep 2026 22:02:36 +0100 Subject: [PATCH 3/7] docs: --since/--until, incremental reruns, finalThrough and opening README and CHANGELOG (Unreleased) for the backfill and history changes. The library example now ends at span().finalThrough, so it writes no partial days either. A test pins the README queue table's waitTime to the spec's `number`. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 74 ++++++++++++++++++++++++++++++++++ README.md | 48 ++++++++++++++++++++-- examples/backfill.mjs | 15 +++++-- test/unit/readme-types.test.ts | 34 ++++++++++++++++ 4 files changed, 164 insertions(+), 7 deletions(-) create mode 100644 test/unit/readme-types.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index e5bca1f..e7a9c7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,80 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- **`themeparks-backfill --since YYYY-MM-DD` and `--until YYYY-MM-DD`.** There + was no way to ask for less than everything: `--since` was an unknown option, + so a key that reaches the whole archive downloaded all of it, every time. Both + days are inclusive and must be real calendar days written `YYYY-MM-DD` + (`2025-02-30`, `2025-1-1` and `20250101` are refused), and `--since` after + `--until` is refused before anything is requested. `--since` applies when a + file is started. A later run accepts the same `--since` or a later one, so a + fixed or a rolling cron line both work, and refuses one earlier than the day + the file was started from, or one that would leave a gap, with what to do + instead of quietly handing back a file that is not what was asked for. + +- **`history.changeRows()` exposes the `opening` state.** The raw history + response carries, per entity, the state in force at the start of the range, + and `changeRows()` threw it away. Without it the time between midnight and an + entity's first change had no known status, so a day rebuilt from raw history + disagreed with the daily summary whenever a ride was still running from the + night before. The result is still the same async generator and yields exactly + what it always did; it now also has `opening`, an object of `HistoryOpening` + keyed by entity id, covering every entity in the response, including one that + did not change all day. It is readable once the response has arrived: iterate + first, or `await changes.load()`. Either way it is one request. The + `HistoryChanges` and `HistoryOpening` types are exported. + +- **`HistorySpan.finalThrough`**: the newest day whose daily row will not change + again, the earlier of `recordedTo` and `retrievableThrough`. + +### Fixed + +- **A finished park now updates on the next run.** A rerun printed + `already complete` and exited 0 without fetching a single new day, so a nightly + cron looked healthy and never updated; the only way to get yesterday was + `--overwrite`, which downloads the whole archive again. A finished file is now + carried forward from the day after its last one, appending only the new days. + An interrupted run still resumes at its page boundary, including a rerun + interrupted before its first page, which would otherwise have started again + from the top of the file's range. + +- **The newest rows of a backfill were partial days, and stayed that way.** A run + ended on `retrievableThrough`, which is usually today: today's row is the day + so far, and the archive records days 2 to 3 behind live data, so the last few + days of every file were still changing when they were written. A run now ends + at `finalThrough`, says so when it holds days back, and the next run adds them + once they are final. Every row in the file is one that will not change. + + **Files written by 8.3.x are corrected once.** Their state file does not say + which of their newest days were final, so the first run of this version + removes the rows dated within seven days of that run's end and fetches those + days again. Every other row is left byte for byte as it was, and a file whose + newest row is older than that is not rewritten at all. A file that lies wholly + inside those seven days, as an anonymous run's does, is downloaded again. The + state file format moves to version 2 for this; version 1 files from this SDK + are upgraded, not refused. + +- **A continued file no longer jumps forward to the key's first day.** When the + day a file continues from is older than the key may read (a cron that did not + run for longer than the key's window, or a key that lost its plan), the run + used to carry on from the key's first day and leave a gap the file then did not + record. It is refused now, the file is left alone, and the message says how to + start again. + +- **A finished file written to a different column layout was appended to.** Only + an unfinished one was refused. A finished one fell through to a fresh start, + which opened the existing file in append mode and wrote the whole archive into + it a second time under a second header, exit 0. It is refused now, the same + way. + +- **A state file whose data file had been deleted was continued**, producing a + file that started part-way through its range and was then recorded as + complete. The park is downloaded again from the start instead. + ## [8.3.1] - 2026-09-28 ### Fixed diff --git a/README.md b/README.md index faf91d8..63fd743 100644 --- a/README.md +++ b/README.md @@ -283,10 +283,11 @@ const DISNEYLAND = '7340550b-c14d-4def-80bb-acdb51d49a66'; const tp = new ThemeParks({ apiKey: process.env.THEMEPARKS_API_KEY }); const history = tp.entity(DISNEYLAND).history; -// What exists, and what your key may read. The same three fields whether the -// id is a park or a single ride. +// What exists, and what your key may read. The same fields whether the id is a +// park or a single ride. const span = await history.span(); -// -> { archiveFrom: '2021-07-03', recordedTo: '2026-09-22', retrievableThrough: '2026-09-23' } +// -> { archiveFrom: '2021-07-03', recordedTo: '2026-09-22', +// retrievableThrough: '2026-09-23', finalThrough: '2026-09-22' } // Pages until the server stops offering a `next`, yielding as it goes. for await (const { entityId, row } of history.days({ @@ -308,6 +309,13 @@ what your key may read, the second is what the archive holds. They differ on every plan below the top one, and asking past the entitlement is how a long run ends in 403s. +**If you write each day once, end at `finalThrough`.** `retrievableThrough` is +usually today, and today's row is the day so far. Recent days can still change +too, because the archive records days 2 to 3 behind live data. `finalThrough` is +the earlier of `recordedTo` and `retrievableThrough`: the newest day whose row +will not change again. Store through that, and fetch the days after it on your +next run. + **`days()` yields, it does not collect.** Nothing accumulates, so the only thing that grows is whatever you write the rows to. @@ -332,6 +340,27 @@ try { `history.changeRows(query)` is the same treatment for `changes`: one flattened stream of `{ entityId, row }` whether you asked a park or a ride. +To rebuild what an entity was doing at any moment you also need the state before +its first change. That is `opening`, keyed by entity id, on the same result: + +```js +const changes = history.changeRows({ date: '2026-09-20' }); +for await (const { entityId, row } of changes) console.log(row.time, entityId, row.status); +for (const [entityId, opening] of Object.entries(changes.opening)) { + // In force from opening.time until this entity's first row. + console.log(entityId, opening.time, opening.status); +} +``` + +Each row holds from its `time` until the next row's, so `opening` plus the rows +cover the whole range with no gap. Without it, a ride still running from the +night before has no known status until its first change. There is an entry for +every entity in the response, including one that did not change all day, and +`opening.observedAt` says when that state was last seen, which can be long before +the range for a ride whose feed stopped. `opening` is readable once the response +has arrived: iterate first, or `await changes.load()`. Either way it is one +request. + ### Or skip the code: there is a command Installing the package puts `themeparks-backfill` on your path. It is the same @@ -346,6 +375,7 @@ export THEMEPARKS_API_KEY=tpw_your_key npx themeparks-backfill --list disney # find your park. This part needs no key. npx themeparks-backfill "magic kingdom" # NDJSON, into the current directory npx themeparks-backfill "Walt Disney World Resort" --format csv --out ./data +npx themeparks-backfill "magic kingdom" --since 2025-01-01 --until 2025-12-31 ``` A park or a **destination**, by name or by id; a destination writes one file per @@ -358,6 +388,18 @@ work it out. It checkpoints against the hourly history budget and exits 75 (`EX_TEMPFAIL`) when that runs out, so a cron or systemd timer retries instead of alerting and the same command continues where it stopped. `--help` has the rest. +**Run it again to update.** A second run of a finished park fetches only the days +that have become final since the last one and appends them, so a nightly cron +keeps the file current. Only final days are written: a run ends at the newest day +the archive has finished recording and says so when it holds newer days back, so +no row in the file changes later. + +**`--since` and `--until`** (`YYYY-MM-DD`, both inclusive) pick the days, instead +of everything your plan reaches. `--since` applies when a file is started: a +later run accepts the same `--since` or a later one, so a fixed or a rolling cron +line both work, and refuses one earlier than the file's first day, or one that +would leave a gap, with what to do instead. `--overwrite` starts the file again. + The CSV is byte-for-byte identical to the Python SDK's, which runs the same command: Magic Kingdom's five-year archive is 94,223 rows and 41 columns from either. diff --git a/examples/backfill.mjs b/examples/backfill.mjs index 545a9b5..8c7c514 100644 --- a/examples/backfill.mjs +++ b/examples/backfill.mjs @@ -14,9 +14,12 @@ * resort is around a hundred times fewer calls than the same data pulled * ride by ride. * - * 2. It bounds the range with span().retrievableThrough, not with what the - * archive holds. Those are different dates on every plan below the top one, - * and asking past the entitlement is how a long backfill ends in 403s. + * 2. It ends at span().finalThrough: the earlier of what your key may read + * (retrievableThrough) and what the archive holds (recordedTo). Asking past + * the entitlement is how a long backfill ends in 403s, and the days past + * recordedTo are not final: today's row is the day so far, and the archive + * records days 2 to 3 behind live data. Stopping there means no row this + * writes will change later. * * 3. It checkpoints. The history budget is hourly, so a spent one can be most * of an hour from resetting. The SDK raises BudgetExhaustedError rather @@ -98,7 +101,11 @@ async function backfillPark(tp, parkId, outDir, format) { const resuming = existsSync(checkpointPath); const hasRows = existsSync(outPath) && statSync(outPath).size > 0; const from = resuming ? readFileSync(checkpointPath, 'utf8').trim() : span.archiveFrom; - const to = span.retrievableThrough; + const to = span.finalThrough; + if (to === null) { + console.error(`${parkId}: nothing final to fetch yet; run again later`); + return 0; + } console.error(`${parkId}: ${from} .. ${to}${resuming ? ' (resumed)' : ''} -> ${outPath}`); diff --git a/test/unit/readme-types.test.ts b/test/unit/readme-types.test.ts new file mode 100644 index 0000000..bf99490 --- /dev/null +++ b/test/unit/readme-types.test.ts @@ -0,0 +1,34 @@ +/** + * The README's queue table against the generated types. + * + * The Python SDK's README said `waitTime` is an `int` while the spec declares a + * JSON `number` and its models use `float`. This README says `number`, which is + * right; this pins it, so a later edit cannot promise whole minutes the API + * does not. + */ + +import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const ROOT = resolve(__dirname, '../..'); + +describe('the README queue table', () => { + it('types waitTime as the spec does: number, never int', () => { + const readme = readFileSync(resolve(ROOT, 'README.md'), 'utf8'); + const rows = readme.split('\n').filter((l) => /^\| `queue\.[A-Z_]+`/u.test(l)); + const waits = rows.filter((l) => l.includes('waitTime')); + expect(waits.map((l) => /`queue\.([A-Z_]+)`/u.exec(l)?.[1])).toEqual([ + 'STANDBY', + 'SINGLE_RIDER', + 'PAID_STANDBY', + ]); + for (const row of waits) expect(row).toContain('`waitTime: number \\| null`'); + expect(readme).not.toMatch(/waitTime:?\s*`?int/u); + + const schema = readFileSync(resolve(ROOT, 'src/_generated/schema.ts'), 'utf8'); + const declared = [...schema.matchAll(/waitTime\??: ([^;]+);/gu)].map((m) => m[1]); + expect(declared.length).toBeGreaterThan(0); + expect(new Set(declared)).toEqual(new Set(['number | null'])); + }); +}); From 7f179f55526ccaa3dd5e9bf925511c27a93464ff Mon Sep 17 00:00:00 2001 From: Jamie Holding Date: Tue, 29 Sep 2026 08:47:13 +0100 Subject: [PATCH 4/7] fix(history): onPage may return a promise; opening is safe to spread days() now awaits whatever onPage returns before requesting the next page, so a checkpoint written there is on disk before anything else can happen. The type is widened to `unknown`, so existing callbacks still compile. changeRows().opening is no longer enumerable, so spreading, Object.keys or a logger walking the result never trips the getter early. After a failed request it says the request failed rather than suggesting load(). The type docs explain `degraded`. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/ergonomic/history.ts | 37 ++++++++++++++++++++++++++----- test/unit/history-opening.test.ts | 9 ++++++++ test/unit/history-paging.test.ts | 25 +++++++++++++++++++++ 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/src/ergonomic/history.ts b/src/ergonomic/history.ts index 0f0aee7..459aca2 100644 --- a/src/ergonomic/history.ts +++ b/src/ergonomic/history.ts @@ -219,7 +219,15 @@ function openingsOf(envelope: EntityHistory): Record { * one request. */ export type HistoryChanges = AsyncGenerator & { - /** The state at the start of the range, per entity id. Throws before the response. */ + /** + * The state at the start of the range, per entity id. Throws, saying how to + * get it, before the response has arrived, and says the request failed after + * one that did. Not enumerable, so spreading or logging the result is safe. + * + * An opening with `degraded: true` is incomplete: the server could not look + * far enough back for this response (`degradedReason` says why), so a field + * it holds may be missing. Ask again in a minute for the full opening. + */ readonly opening: Record; /** Make the request now, if it has not been made, and resolve to this object. */ load(): Promise; @@ -253,8 +261,12 @@ export interface HistoryPage { } export interface PageOptions { - /** Called after every row of a page has been yielded. See {@link HistoryPage}. */ - onPage?: (page: HistoryPage) => void; + /** + * Called after every row of a page has been yielded. See {@link HistoryPage}. + * It may return a promise: the next page is not requested until it settles, + * so a checkpoint written here is on disk before anything else happens. + */ + onPage?: (page: HistoryPage) => unknown; } export type DaysOptions = HistoryQuery & BudgetOptions & PageOptions; @@ -319,7 +331,10 @@ export class HistoryApi { const next = envelope.next; // AFTER the rows, never before: a caller checkpointing on this has to be // able to trust that everything the page held is already written. - options.onPage?.({ + // AWAITED, so a caller can make its checkpoint durable before the next + // page is requested: an interruption then never lands between a page + // being written and the record that it was. + await options.onPage?.({ from: envelope.range.from, to: envelope.range.to, next: next === '' ? null : next, @@ -354,6 +369,7 @@ export class HistoryApi { // flight) cost one request between them. let pending: Promise | null = null; let envelope: EntityHistory | null = null; + let failure: unknown = null; const fetchOnce = (): Promise => { pending ??= this.raw.getEntityHistory(this.entityId, toQuery(options)).then( (value) => { @@ -361,7 +377,8 @@ export class HistoryApi { return value; }, (error: unknown) => { - throw asBudgetError(error, maxWaitMs); + failure = asBudgetError(error, maxWaitMs); + throw failure; }, ); return pending; @@ -373,9 +390,17 @@ export class HistoryApi { const changes = rows() as HistoryChanges; Object.defineProperties(changes, { + // NOT ENUMERABLE, so spreading, `Object.keys` or a logger walking the + // object never trips the getter before the response has arrived. opening: { - enumerable: true, + enumerable: false, get(): Record { + if (failure !== null) { + const why = failure instanceof Error ? failure.message : String(failure); + throw new Error(`the request for this history failed, so there is no opening: ${why}`, { + cause: failure, + }); + } if (envelope === null) { throw new Error( 'the response has not arrived yet: iterate first, or ' + diff --git a/test/unit/history-opening.test.ts b/test/unit/history-opening.test.ts index bbb98c7..2266b56 100644 --- a/test/unit/history-opening.test.ts +++ b/test/unit/history-opening.test.ts @@ -192,6 +192,15 @@ describe('changeRows() exposes opening', () => { const changes: HistoryChanges = tp.entity(SPACE_MOUNTAIN).history.changeRows({ date: 'x' }); await expect(changes.load()).rejects.toThrow(); await expect(collect(changes)).rejects.toThrow(); + // Not "has not arrived yet, call load()": load() was called, and failed. + expect(() => changes.opening).toThrow(/failed/u); + }); + + it('can be spread and listed before the response without throwing', () => { + const changes = client(RAW).entity(SPACE_MOUNTAIN).history.changeRows({ date: '2026-09-26' }); + expect(() => ({ ...changes })).not.toThrow(); + expect(Object.keys(changes)).not.toContain('opening'); + expect(() => JSON.stringify(changes)).not.toThrow(); }); }); diff --git a/test/unit/history-paging.test.ts b/test/unit/history-paging.test.ts index 6376492..310baa4 100644 --- a/test/unit/history-paging.test.ts +++ b/test/unit/history-paging.test.ts @@ -355,6 +355,31 @@ describe('what a daily row is labelled with', () => { }); describe('the page hook', () => { + it('waits for a promise it returns before asking for the next page', async () => { + // A checkpoint written in the hook has to be durable before the next + // request: an interruption during that request must find it on disk. + const page1 = await loadFixture('mk_park_daily_page1.json'); + const page2 = await loadFixture('mk_park_daily_page2.json'); + const order: string[] = []; + let call = 0; + const fetchFn = vi.fn(() => { + call += 1; + order.push(`fetch:${String(call)}`); + return Promise.resolve(json(call === 1 ? page1 : page2)); + }); + await collect( + client(fetchFn) + .entity('mk') + .history.days({ + onPage: async (p) => { + await new Promise((r) => setTimeout(r, 20)); + order.push(`saved:${p.to}`); + }, + }), + ); + expect(order).toEqual(['fetch:1', 'saved:2026-08-31', 'fetch:2', 'saved:2026-09-20']); + }); + it('reports the range and the next page the server gave', async () => { const page1 = await loadFixture('mk_park_daily_page1.json'); const page2 = await loadFixture('mk_park_daily_page2.json'); From 1b9e5e86c6f7e2dee92d0765d2844fd9f79e58de Mon Sep 17 00:00:00 2001 From: Jamie Holding Date: Tue, 29 Sep 2026 08:55:12 +0100 Subject: [PATCH 5/7] fix(backfill): checkpoint every page, so an interrupted run never duplicates The state file was written only when a run ended or failed in a way it caught. A Ctrl-C, a SIGTERM or a kill during a multi-page download left the previous state on disk, so the next run fetched every page since and appended it a second time. - The state is written atomically (temp file, then rename) after every complete page, and once before the first request, so an extending run is marked unfinished before it asks for anything. - Each checkpoint waits for the rows before it to reach the file and records the file's size. A resume first cuts the file back to that size, so a page cut off mid-write is fetched again rather than appended twice. A file shorter than its checkpoint is refused. A state with no size that resumes on its last day drops that day first. - days() awaits onPage, so the checkpoint is on disk before the next request. - A lock beside the state file refuses a second run on the same output while one is running; a lock left by a process that is gone is taken over. - SIGINT and SIGTERM release the locks and exit (130, 143). Nothing else needs flushing: the last checkpoint is already on disk. Tested by abandoning a run mid-page, as a kill would, and resuming it. The stream-error test now reaches the stream (the lock is taken first), and five stale mutants are updated alongside four new ones. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/backfill-cli.ts | 24 ++- src/backfill.ts | 256 ++++++++++++++++++++----- test/mutation/mutants.json | 50 +++-- test/unit/backfill-incremental.test.ts | 195 ++++++++++++++++++- test/unit/backfill.test.ts | 35 ++++ 5 files changed, 489 insertions(+), 71 deletions(-) diff --git a/src/backfill-cli.ts b/src/backfill-cli.ts index 5f6dec6..39283c5 100644 --- a/src/backfill-cli.ts +++ b/src/backfill-cli.ts @@ -17,14 +17,24 @@ * * A file whose only job is to run has no condition to get wrong. */ -import { run } from './backfill.js'; +import { releaseLocks, run } from './backfill.js'; -// Ctrl-C says how to continue, like the Python SDK. 130 is the shell's convention -// for SIGINT, and the state files on disk already say where each park got to. -process.on('SIGINT', () => { - process.stderr.write('\nstopped. Run the same command again to continue.\n'); - process.exit(130); -}); +// Ctrl-C, or a scheduler's SIGTERM, says how to continue, like the Python SDK. +// Exiting straight away is safe: the state file is rewritten atomically after +// every complete page, and the next run cuts off any rows written after that +// checkpoint before resuming, so an interrupted page is fetched again rather +// than appended twice. The locks are released so the next run need not wait +// to find them stale. 130 and 143 are the shell's conventions (128 + signal). +for (const [signal, code] of [ + ['SIGINT', 130], + ['SIGTERM', 143], +] as const) { + process.on(signal, () => { + releaseLocks(); + process.stderr.write('\nstopped. Run the same command again to continue.\n'); + process.exit(code); + }); +} run().then( (code) => { diff --git a/src/backfill.ts b/src/backfill.ts index f0a0b8c..4996616 100644 --- a/src/backfill.ts +++ b/src/backfill.ts @@ -55,6 +55,7 @@ import { renameSync, rmSync, statSync, + truncateSync, writeFileSync, writeSync, } from 'node:fs'; @@ -545,6 +546,12 @@ interface BackfillState { * `--since` before that day keep running. Absent from version-1 files. */ since?: string | null; + /** + * The size of the data file when this state was written. Everything past it + * was appended after the checkpoint (a page cut off by an interruption) and is + * truncated before a resume, so a resume never appends a day twice. + */ + bytes?: number; } function readState(path: string): BackfillState | null { @@ -584,8 +591,73 @@ function stateMismatch(state: BackfillState, format: string): string | null { return null; } +/** + * Write the state file ATOMICALLY: a temporary file beside it, then one rename. + * A process killed mid-write leaves the old state or the new one, never half a + * JSON document that the next run would read as no state at all. + */ function writeState(path: string, state: BackfillState): void { - writeFileSync(path, `${JSON.stringify(state, null, 0)}\n`, 'utf8'); + const scratch = `${path}.tmp`; + writeFileSync(scratch, `${JSON.stringify(state, null, 0)}\n`, 'utf8'); + renameSync(scratch, path); +} + +// --------------------------------------------------------------------------- +// One run per output file at a time. +// --------------------------------------------------------------------------- + +const heldLocks = new Set(); + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + // EPERM: it exists, it is just not ours to signal. + return (error as NodeJS.ErrnoException).code === 'EPERM'; + } +} + +/** + * Take the lock beside a park's state file, or return the pid holding it. + * + * Two runs appending to one file interleave their rows and each records its own + * checkpoint, so the file ends up with duplicates and a state that describes + * neither. A cron that fires while yesterday's long run is still going is the + * ordinary way to get there. The lock holds the owner's pid; one left by a + * process that no longer exists (a SIGKILL, a reboot) is taken over. + */ +function acquireLock(lockPath: string): number | null { + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + writeFileSync(lockPath, `${String(process.pid)}\n`, { flag: 'wx' }); + heldLocks.add(lockPath); + return null; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + let owner = Number.NaN; + try { + owner = Number(readFileSync(lockPath, 'utf8').trim()); + } catch { + // Gone between the two calls: try again. + } + if (Number.isInteger(owner) && owner > 0 && isAlive(owner)) return owner; + rmSync(lockPath, { force: true }); + } + } + return -1; +} + +function releaseLock(lockPath: string): void { + if (heldLocks.delete(lockPath)) rmSync(lockPath, { force: true }); +} + +/** + * Release every lock this process holds. For a signal handler: the state on disk + * is already the last checkpoint, so the locks are all that is left to tidy. + */ +export function releaseLocks(): void { + for (const lockPath of [...heldLocks]) releaseLock(lockPath); } // --------------------------------------------------------------------------- @@ -722,6 +794,10 @@ export function decide( rmSync(statePath, { force: true }); } let state = opts.overwrite ? null : readState(statePath); + if (state !== null) { + const cut = cutToCheckpoint(outPath, state, opts.format); + if (cut !== null) return cut; + } const hasRows = (): boolean => existsSync(outPath) && statSync(outPath).size > 0; let fileHasRows = hasRows(); @@ -766,6 +842,39 @@ export function decide( return continueFile(outPath, state, opts, archiveFrom, end); } +/** + * Put the data file back to the size it had at its last checkpoint. + * + * Rows are appended as they arrive and the checkpoint is written once a page is + * complete, so an interruption (Ctrl-C, SIGTERM, a kill, a crash) can leave + * part of a page after the last checkpoint. Resuming at the checkpoint's day + * fetches that page again, and appending it would duplicate every row already + * there. Truncating to the recorded size first makes the resume exact. Null to + * proceed, or an exit code when the file is SHORTER than recorded, which means + * something else changed it and nothing here can say which days it lost. + */ +function cutToCheckpoint(outPath: string, state: BackfillState, format: string): number | null { + if (state.complete || typeof state.bytes !== 'number' || !existsSync(outPath)) return null; + if (stateMismatch(state, format) !== null) return null; + const size = statSync(outPath).size; + if (size === state.bytes) return null; + if (size < state.bytes) { + process.stderr.write( + ` ${basename(outPath)} is shorter than when this command last recorded it ` + + `(${String(size)} bytes, not ${String(state.bytes)}), so rows it held are gone.\n` + + ` --overwrite download this park again\n` + + ` or move both files aside and run again\n`, + ); + return 1; + } + truncateSync(outPath, state.bytes); + process.stderr.write( + ` ${basename(outPath)}: removed the rows written after the last checkpoint ` + + `(an interrupted page); they are fetched again\n`, + ); + return null; +} + /** A park with no file yet: from `--since`, or wherever the archive starts. */ function firstRun( outPath: string, @@ -833,6 +942,12 @@ function continueFile( const continueAt = state.resumeFrom ?? state.lastDay ?? state.start ?? archiveFrom; const refused = rangeFits(outPath, state, opts, archiveFrom, String(continueAt)); if (refused !== null) return refused; + // A resume at `lastDay` refetches a day the file already holds. Only a state + // with no byte count can get here (one written before checkpoints recorded + // it), so that day's rows are removed first and the resume stays exact. + if (state.resumeFrom === null && state.lastDay !== null && typeof state.bytes !== 'number') { + trimAfter(outPath, opts.format, daysBefore(state.lastDay, 1)); + } return { ...carried, start: continueAt, extending: false }; } @@ -1088,6 +1203,32 @@ export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions) const statePath = statePathFor(opts.outDir, park.id, opts.format); const end = runEnd(span, opts); + const lockPath = `${statePath}.lock`; + const owner = acquireLock(lockPath); + if (owner !== null) { + process.stderr.write( + ` another run is writing ${basename(outPath)}` + + `${owner > 0 ? ` (process ${String(owner)})` : ''}. Wait for it to finish, or ` + + `stop it, then run again\n`, + ); + return 1; + } + try { + return await runPark(history, park, opts, span, { outPath, statePath, end }); + } finally { + releaseLock(lockPath); + } +} + +/** Everything after the lock: decide, stream, checkpoint, record. */ +async function runPark( + history: ReturnType['history'], + park: Park, + opts: RunOptions, + span: HistorySpan, + paths: { outPath: string; statePath: string; end: string | null }, +): Promise { + const { outPath, statePath, end } = paths; const decided = decide(outPath, statePath, opts, span.archiveFrom, end); if (typeof decided === 'number') return decided; let { start } = decided; @@ -1135,6 +1276,14 @@ export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions) handle.on('error', (error: Error) => { streamError = error; }); + // Settles once everything written so far has reached the file. A checkpoint + // waits on it, so the size it records covers exactly the rows before it. + let flushed: Promise = Promise.resolve(); + const append = (text: string): void => { + flushed = new Promise((done) => { + handle.write(text, () => done()); + }); + }; // ONE header decision for the whole park. Python built the writer inside the // retried closure with `written === 0` in the predicate, and the 403 recovery // runs precisely when that is true — so every CSV on every plan short of the @@ -1143,44 +1292,58 @@ export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions) // A UTF-8 BOM, so Excel on Windows does not read the local code page and render // `Walt Disney World® Resort` as mojibake. The primary reader of this file is a // spreadsheet. Written with the header, so a resume never adds a second. - handle.write(`\ufeff${CSV_COLUMNS.join(',')}\n`); + append(`\ufeff${CSV_COLUMNS.join(',')}\n`); } let written = 0; let lastDay: string | null = null; - let resumeFrom: string | null = null; let skipped = false; let outOfReach = false; - // The day this run's request started on, which is where a rerun has to carry - // on if the run fails before its first page is finished. - let firstDay: string | null = null; + // The first day the file covers. Moves to the key's floor when a run that has + // written nothing yet is told its key starts later. + let fileStart = priorStart ?? start; + + const stateNow = (complete: boolean, resumeFrom: string | null): BackfillState => ({ + sdk: SDK_NAME, + sdkVersion: PACKAGE_VERSION, + stateVersion: STATE_VERSION, + columns: columnsFingerprint(opts.format), + format: opts.format, + start: fileStart, + end, + lastDay: lastDay ?? priorLastDay, + resumeFrom, + complete, + since: since ?? fileStart, + bytes: existsSync(outPath) ? statSync(outPath).size : 0, + }); /** - * Where a rerun should continue, for the state file. + * THE CHECKPOINT, written after every complete page and once before the first + * request, so the state on disk always says where to carry on. It used to be + * written only at the end of a run or on an error this code caught, so a + * Ctrl-C, a SIGTERM or a kill mid-download left the state of the run before: + * the next run re-fetched every page since and appended it a second time. * - * The next page's start once a page is done. Before that, with rows written, - * null: `lastDay` is the fallback and costs one duplicated day. Before ANY - * row, the day this run began on. That last case had nothing recorded, which - * was harmless while only a first run could reach it -- a rerun of a file - * with no rows starts from the top anyway -- and is not now that a finished - * file is carried forward. A rerun interrupted before its first page would - * otherwise fall back to the file's start and append the whole range again. + * The rows are flushed first and the file's size recorded with them, so the + * next run can cut off anything written after this point before resuming. */ - const resumePoint = (): string | null => { - if (resumeFrom !== null) return resumeFrom; - if (lastDay !== null) return null; - return firstDay; + const checkpoint = async (resumeFrom: string | null): Promise => { + await flushed; + if (streamError) throw streamError; + writeState(statePath, stateNow(false, resumeFrom)); }; const stream = async (from: string | null): Promise => { - firstDay = from; // `exactOptionalPropertyTypes` means an explicit undefined is not the same // as an absent key, so the query is built rather than spread with nulls. - const query: { from?: string; to?: string; onPage: (page: HistoryPage) => void } = { - // The checkpoint. Fires once a page's rows are all written, carrying the - // day the NEXT page starts on, so a resume asks for nothing twice. - onPage: (page) => { - resumeFrom = page.next === null ? null : nextPageFrom(page.next); + const query: { from?: string; to?: string; onPage: (page: HistoryPage) => Promise } = { + // Fires once a page's rows are all written, carrying the day the NEXT page + // starts on, and the next page is not requested until it has settled. The + // last page has no checkpoint: the run records itself complete right after. + onPage: async (page) => { + const nextFrom = page.next === null ? null : nextPageFrom(page.next); + if (nextFrom !== null) await checkpoint(nextFrom); }, }; if (from !== null) query.from = from; @@ -1190,7 +1353,7 @@ export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions) // this the loop keeps "writing" into a broken stream for the rest of the // archive and only the flush would notice. if (streamError) throw streamError; - handle.write(opts.format === 'csv' ? `${csvLine(park, entry)}\n` : ndjsonLine(park, entry)); + append(opts.format === 'csv' ? `${csvLine(park, entry)}\n` : ndjsonLine(park, entry)); written += 1; const day = (entry.row as { date?: string }).date ?? null; // MAX, not last-seen. Entities arrive name-ordered with independent day @@ -1214,21 +1377,9 @@ export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions) */ const finish = (): Promise => flushAndClose(handle, () => streamError); - const record = (complete: boolean): void => { - writeState(statePath, { - sdk: SDK_NAME, - sdkVersion: PACKAGE_VERSION, - stateVersion: STATE_VERSION, - columns: columnsFingerprint(opts.format), - format: opts.format, - start: priorStart ?? start, - end, - lastDay: lastDay ?? priorLastDay, - resumeFrom: complete ? null : resumePoint(), - complete, - since: since ?? priorStart ?? start, - }); - }; + // Before the first request: an extending run is no longer complete from here + // on, and a fresh one has a state to resume from even if it dies at once. + await checkpoint(start); try { try { @@ -1243,7 +1394,7 @@ export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions) // state file cannot describe, so the file would claim days it does not // hold. It happens when a cron has not run for longer than the key's // window, or the key lost its plan. Refused, with the file left alone. - if (resumed) { + if (resumed && priorLastDay !== null) { if (start === null || floor <= start) throw error; process.stderr.write( ` this key reaches back to ${floor}, but ${basename(outPath)} continues from ` + @@ -1258,6 +1409,7 @@ export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions) ` this key reaches back to ${floor}, not ${String(start)} — starting there\n`, ); start = floor; + fileStart = floor; if (isEmptyWindow(floor, end)) { process.stderr.write( ` nothing in your window: this park's data ends ${String(end)} — skipping\n`, @@ -1270,17 +1422,18 @@ export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions) } } catch (error) { await finish(); + // The state on disk is the last checkpoint, which is exactly where a rerun + // should carry on; nothing is recorded here. if (error instanceof BudgetExhaustedError) { - record(false); - if (written === 0 && !resumed) rmSync(outPath, { force: true }); + if (written === 0 && !resumed) { + rmSync(outPath, { force: true }); + rmSync(statePath, { force: true }); + } process.stderr.write( ` budget spent after ${String(written)} rows; rerun the same command to continue\n`, ); return EX_TEMPFAIL; } - // Any other failure still records where it got to, or the next run starts - // over and appends a second partial copy. - if (lastDay !== null) record(false); if (written === 0 && resumed) { // An earlier run's rows are real and are not ours to remove. process.stderr.write(` the rows already downloaded are kept\n`); @@ -1292,7 +1445,10 @@ export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions) // destination the customer counts six files and never sees which one is // empty. Written rows stay: they are real, and the state file beside them // says where to carry on. - if (written === 0) rmSync(outPath, { force: true }); + if (written === 0) { + rmSync(outPath, { force: true }); + rmSync(statePath, { force: true }); + } throw error; } @@ -1302,15 +1458,15 @@ export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions) if (outOfReach) return 1; if (skipped && written === 0) { if (resumed) { - // Same rule: keep the file, record where it got to, and say it did not finish. - record(false); + // Same rule: keep the file, and say it did not finish. The checkpoint on + // disk still says where it would carry on from. return 1; } rmSync(outPath, { force: true }); rmSync(statePath, { force: true }); return 0; } - record(true); + writeState(statePath, stateNow(true, null)); process.stderr.write(` done: ${String(written)} rows -> ${outPath}\n`); return 0; } diff --git a/test/mutation/mutants.json b/test/mutation/mutants.json index 0a08532..0c267b2 100644 --- a/test/mutation/mutants.json +++ b/test/mutation/mutants.json @@ -60,8 +60,8 @@ { "name": "the checkpoint comes from the newest row", "file": "src/backfill.ts", - "find": " resumeFrom = page.next === null ? null : nextPageFrom(page.next);", - "replace": " resumeFrom = lastDay;", + "find": " const nextFrom = page.next === null ? null : nextPageFrom(page.next);", + "replace": " const nextFrom = lastDay;", "why": "An entity that stopped reporting has no rows for the tail days of its page, so resuming there re-downloads days already written." }, { @@ -74,8 +74,8 @@ { "name": "the CSV loses its BOM", "file": "src/backfill.ts", - "find": " handle.write(`\\ufeff${CSV_COLUMNS.join(',')}\\n`);", - "replace": " handle.write(`${CSV_COLUMNS.join(',')}\\n`);", + "find": " append(`\\ufeff${CSV_COLUMNS.join(',')}\\n`);", + "replace": " append(`${CSV_COLUMNS.join(',')}\\n`);", "why": "Excel on Windows reads the local code page and mangles every registered mark and accent." }, { @@ -137,8 +137,8 @@ { "name": "the page hook fires before its rows", "file": "src/ergonomic/history.ts", - "find": " yield* dailyEntries(envelope);\n const next = envelope.next;\n // AFTER the rows, never before: a caller checkpointing on this has to be\n // able to trust that everything the page held is already written.\n options.onPage?.(", - "replace": " const next = envelope.next;\n options.onPage?.(", + "find": " yield* dailyEntries(envelope);\n const next = envelope.next;", + "replace": " const next = envelope.next;", "why": "A consumer that died mid-page would record a checkpoint past rows it never wrote. Losing rows is worse than duplicating them." }, { @@ -226,11 +226,11 @@ "why": "Continuing writes a file that starts part-way through its range and records it as complete." }, { - "name": "a rerun interrupted before its first page records no resume point", + "name": "no checkpoint before the first request", "file": "src/backfill.ts", - "find": "return firstDay;", - "replace": "return null;", - "why": "The next run fell back to the file's start and appended the whole range again." + "find": " await checkpoint(start);\n", + "replace": "", + "why": "A run killed before its first page left no state, or an extending run's state still said complete; the next run started over or skipped the days." }, { "name": "an 8.3 file keeps its partial tail", @@ -256,7 +256,7 @@ { "name": "a continued file jumps forward to the key's floor", "file": "src/backfill.ts", - "find": " if (resumed) {", + "find": " if (resumed && priorLastDay !== null) {", "replace": " if (false) {", "why": "Carrying on from the key's first day instead of the file's next day leaves a gap the file then claims not to have." }, @@ -280,6 +280,34 @@ "find": "return a < b ? a : b;", "replace": "return a > b ? a : b;", "why": "The later day is retrievableThrough on most keys: today, whose row is not final." + }, + { + "name": "onPage is not awaited", + "file": "src/ergonomic/history.ts", + "find": " await options.onPage?.({", + "replace": " void options.onPage?.({", + "why": "The next page was requested while the checkpoint was still being written, so a kill in between recorded the wrong page." + }, + { + "name": "a resume does not cut back to the checkpoint", + "file": "src/backfill.ts", + "find": " truncateSync(outPath, state.bytes);", + "replace": "", + "why": "Rows written after the last checkpoint by an interrupted page were kept, and the resume appended the same page again." + }, + { + "name": "a checkpoint does not wait for the rows", + "file": "src/backfill.ts", + "find": " await flushed;\n", + "replace": "", + "why": "The size recorded would not cover the rows before it, and the resume would cut some of them off." + }, + { + "name": "overlapping runs are not locked out", + "file": "src/backfill.ts", + "find": " if (owner !== null) {", + "replace": " if (false) {", + "why": "Two runs appending to one file interleave rows and each records its own checkpoint." } ] } diff --git a/test/unit/backfill-incremental.test.ts b/test/unit/backfill-incremental.test.ts index 6144499..c34cacb 100644 --- a/test/unit/backfill-incremental.test.ts +++ b/test/unit/backfill-incremental.test.ts @@ -29,7 +29,9 @@ import { mkdirSync, mkdtempSync, readFileSync, + readdirSync, rmSync, + truncateSync, statSync, writeFileSync, } from 'node:fs'; @@ -44,6 +46,7 @@ import { csvLine, main, ndjsonLine, + releaseLocks, statePathFor, trimAfter, } from '../../src/backfill'; @@ -94,6 +97,15 @@ class Archive implements Pick { calls: [string, string][] = []; /** Throw BudgetExhaustedError on the Nth page request (1-based), or never. */ budgetOnPage: number | null = null; + /** + * Stop dead on the Nth page (1-based) after this many rows, never returning: + * a process killed mid-page, as far as anything on disk can tell. + */ + hangOnPage: number | null = null; + hangAfterRows = 0; + /** Resolves once the stub has hung. */ + hung: Promise; + private signalHung: () => void = () => undefined; pagesServed = 0; names: Record = {}; @@ -103,7 +115,20 @@ class Archive implements Pick { public through = '2026-09-28', public floor: string | null = null, public entities: string[] = ['ent-a', 'ent-b'], - ) {} + ) { + this.hung = new Promise((resolve) => { + this.signalHung = resolve; + }); + } + + /** Hang on the Nth page served from now, after `rows` rows of it. */ + armHang(pagesFromNow: number, rows: number): void { + this.hangOnPage = this.pagesServed + pagesFromNow; + this.hangAfterRows = rows; + this.hung = new Promise((resolve) => { + this.signalHung = resolve; + }); + } /** Time passes: the archive and today both move on. */ advance(days: number): void { @@ -138,8 +163,14 @@ class Archive implements Pick { }); } const pageEnd = minDay(addDays(pageStart, PAGE_DAYS - 1), last); + let rowsThisPage = 0; for (const entity of this.entities) { for (let day = pageStart; day <= pageEnd; day = addDays(day, 1)) { + if (this.hangOnPage === this.pagesServed && rowsThisPage === this.hangAfterRows) { + this.signalHung(); + await new Promise(() => undefined); + } + rowsThisPage += 1; yield await Promise.resolve(this.entry(entity, day)); } } @@ -148,7 +179,7 @@ class Archive implements Pick { following <= last ? `https://api.themeparks.wiki/v1/entity/p/history/daily?from=${following}&to=${last}` : null; - options.onPage?.({ from: pageStart, to: pageEnd, next }); + await options.onPage?.({ from: pageStart, to: pageEnd, next }); // as days() does pageStart = following; } } @@ -421,7 +452,11 @@ describe('a rerun adds new days', () => { expect(readFileSync(dataPath())).toEqual(before); expect(stderr()).toContain('gap'); expect(stderr()).toContain('--overwrite'); - expect(state()).toMatchObject({ end: '2026-09-26', complete: true }); + // The run recorded itself unfinished before its first request, so the next + // run meets the same refusal rather than a file that looks complete. + expect(state()).toMatchObject({ complete: false, resumeFrom: '2026-09-27' }); + expect(await run(archive)).toBe(1); + expect(readFileSync(dataPath())).toEqual(before); }); }); @@ -866,6 +901,160 @@ describe('a finished file written to another contract is refused', () => { }); }); +// --------------------------------------------------------------------------- +// Interrupted runs: Ctrl-C, SIGTERM, a kill. Nothing gets to clean up. +// --------------------------------------------------------------------------- + +/** + * Start a run, let it hang mid-page, and abandon it as a killed process would + * be abandoned: whatever reached the file stays there, and the only other thing + * that happens is the signal handler releasing the locks. + */ +async function killMidPage( + archive: Archive, + pagesFromNow: number, + rows: number, + format: 'ndjson' | 'csv' = 'ndjson', +): Promise { + archive.armHang(pagesFromNow, rows); + void run(archive, format); + await archive.hung; + await new Promise((r) => setTimeout(r, 50)); // what was written reaches the file + releaseLocks(); + archive.hangOnPage = null; +} + +function contiguous(written: OutRow[], from: string, to: string): void { + const days = new Set(written.map((r) => r.date)); + const expected = (Date.parse(to) - Date.parse(from)) / 86_400_000 + 1; + expect(days.size, 'a gap or an overlap').toBe(expected); + expect(minDate(written)).toBe(from); + expect(maxDate(written)).toBe(to); +} + +describe('a run killed part-way through', () => { + it.each(['ndjson', 'csv'] as const)( + 'resumes mid-archive without appending a row twice (%s)', + async (format) => { + const archive = new Archive(); + await killMidPage(archive, 3, 10, format); + const saved = state(format); + // Pages one and two are checkpointed; page three was cut off. + expect(saved).toMatchObject({ complete: false, resumeFrom: '2026-08-02' }); + expect(statSync(dataPath(format)).size).toBeGreaterThan(saved.bytes as number); + + expect(await run(archive, format)).toBe(0); + expect(stderr()).toContain('removed the rows written after the last checkpoint'); + expectEveryRowFinalAndUnique(rows(format)); + contiguous(rows(format), '2026-06-01', '2026-09-26'); + if (format === 'csv') { + expect(readFileSync(dataPath('csv'), 'utf8').split('parkId,')).toHaveLength(2); + } + }, + ); + + it('records an extending run as unfinished before it asks for anything', async () => { + const archive = new Archive(); + await run(archive); + archive.advance(40); + await killMidPage(archive, 1, 5); + expect(state()).toMatchObject({ complete: false, resumeFrom: '2026-09-27' }); + expect(await run(archive)).toBe(0); + expectEveryRowFinalAndUnique(rows()); + contiguous(rows(), '2026-06-01', '2026-11-05'); + }); + + it('leaves a state to resume from when killed before its first row', async () => { + const archive = new Archive(); + await killMidPage(archive, 1, 0); + expect(state()).toMatchObject({ complete: false, resumeFrom: '2026-06-01', bytes: 0 }); + expect(await run(archive)).toBe(0); + contiguous(rows(), '2026-06-01', '2026-09-26'); + }); + + it("moves the file's start to the key's floor when killed before its first row there", async () => { + const archive = new Archive(); + archive.floor = '2026-08-01'; + await killMidPage(archive, 1, 0); + expect(await run(archive)).toBe(0); + expect(state().start).toBe('2026-08-01'); + contiguous(rows(), '2026-08-01', '2026-09-26'); + }); + + it('refuses a file shorter than its checkpoint says', async () => { + const archive = new Archive(); + await killMidPage(archive, 3, 10); + truncateSync(dataPath(), (state().bytes as number) - 10); + expect(await run(archive)).toBe(1); + expect(stderr()).toContain('shorter than when this command last recorded it'); + }); + + it('drops the day a state with no byte count would resume on, before refetching it', async () => { + // A state from before checkpoints recorded the file's size, interrupted in + // its first page: it resumes on `lastDay`, which the file already holds. + const archive = new Archive('2026-06-01', '2026-07-10', '2026-07-10'); + await writeAs83('ndjson', archive); + writeFileSync( + statePathFor(dir, 'p', 'ndjson'), + JSON.stringify({ + sdk: 'js', + sdkVersion: '8.4.0', + stateVersion: STATE_VERSION, + columns: '', + format: 'ndjson', + start: '2026-06-01', + end: '2026-09-26', + lastDay: '2026-07-10', + resumeFrom: null, + complete: false, + }), + ); + const later = new Archive(); + expect(await run(later)).toBe(0); + expect(later.calls).toEqual([['2026-07-10', '2026-09-26']]); + expectEveryRowFinalAndUnique(rows()); + contiguous(rows(), '2026-06-01', '2026-09-26'); + }); + + it('writes the state file whole, leaving no scratch file behind', async () => { + const archive = new Archive(); + await run(archive); + expect(readdirSync(dir).filter((f) => f.endsWith('.tmp') || f.endsWith('.lock'))).toEqual([]); + expect(() => state()).not.toThrow(); + }); +}); + +describe('one run per output file', () => { + it('refuses a second run while the first is still writing', async () => { + const first = new Archive(); + first.armHang(2, 3); + void run(first); + await first.hung; + const second = new Archive(); + expect(await run(second)).toBe(1); + expect(stderr()).toContain('another run is writing p.ndjson'); + expect(second.calls).toEqual([]); + releaseLocks(); + }); + + it('takes over a lock left by a process that is gone', async () => { + writeFileSync(`${statePathFor(dir, 'p', 'ndjson')}.lock`, '2147483646\n'); + const archive = new Archive(); + expect(await run(archive)).toBe(0); + expect(existsSync(`${statePathFor(dir, 'p', 'ndjson')}.lock`)).toBe(false); + }); + + it('does not lock one format out with the other', async () => { + const archive = new Archive(); + archive.armHang(2, 3); + void run(archive, 'ndjson'); + await archive.hung; + archive.hangOnPage = null; + expect(await run(new Archive(), 'csv')).toBe(0); + releaseLocks(); + }); +}); + describe('a state file with no data file beside it', () => { it('downloads the park again rather than continuing into a new file', async () => { // The state describes a file that is no longer there. Continuing would write diff --git a/test/unit/backfill.test.ts b/test/unit/backfill.test.ts index b41bf44..1b4e273 100644 --- a/test/unit/backfill.test.ts +++ b/test/unit/backfill.test.ts @@ -920,6 +920,41 @@ describe('a failed write is never reported as success', () => { ); } }); + + it('an unwritable data file in a writable directory fails the park cleanly', async () => { + // Node hands `end`'s callback the stream's error as `cb(err)`; `finish()` took + // no arguments and called `done()` regardless, so a failed stream resolved, + // `record(true)` ran, and the command printed `done: N rows` and exited 0 with + // the file truncated. On ENOSPC mid-download that is a short file marked + // complete, which no rerun would ever continue. + const page = await loadFixture('mk_park_daily_page2.json'); + const coverage = await loadFixture('mk_history_coverage.json'); + const fetchFn = vi.fn((input: unknown) => { + const url = String(input); + const body = url.includes('/history/coverage') + ? coverage + : url.includes('/history/daily') + ? page + : DESTINATIONS; + return Promise.resolve( + new Response(JSON.stringify(body), { headers: { 'content-type': 'application/json' } }), + ); + }); + // The directory takes the lock and the state; only the data file refuses. + // The stream's open fails asynchronously, so without an 'error' listener + // this is an uncaught exception rather than a failed park. + writeFileSync(join(dir, `${MK}.ndjson`), ''); + chmodSync(join(dir, `${MK}.ndjson`), 0o400); + const code = await main([MK, '--out', dir, '--api-key', 'k'], { fetch: fetchFn }); + expect(code).not.toBe(0); + // And no state file claiming the park finished. + const state = statePathFor(dir, MK, 'ndjson'); + if (existsSync(state)) { + expect((JSON.parse(readFileSync(state, 'utf8')) as { complete: boolean }).complete).toBe( + false, + ); + } + }); }); describe('an earlier run’s rows are never deleted', () => { From 0e0ba4d63b58d3b4b0fb67bae2db3c831950a5ff Mon Sep 17 00:00:00 2001 From: Jamie Holding Date: Tue, 29 Sep 2026 09:08:33 +0100 Subject: [PATCH 6/7] fix(backfill): agree with the Python SDK on checkpoints, --since and wording - The checkpoint field is `size`, as in Python. A rerun cuts the file back to it and says "discarding the last N bytes of X: written after the last checkpoint, and fetched again now". A file shorter than its checkpoint is refused in the same words. - A state with no size is cut back by date: a finished file to its end, an unfinished one to its page boundary, or a whole page (31 days) before its last day when it has none. Resuming at the last day, as before, lost the days of entities that had not reached it. - `start` is the first day the file covers, after the key's floor; `since` is the start it was asked for. A --since before the first day is refused unless it is that asked-for start, with Python's wording. - The lock and anonymous-access messages match Python's, and the anonymous notice says only final days are written, usually 4 or 5 per park. - An 8.3 file being upgraded says what is done to it, and an --until the file already holds no longer blames the key's plan. Tests for the two mutants an independent sweep found surviving (the cut boundary, and a run that adds no rows keeping lastDay); 45/45 mutants killed. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/backfill.ts | 219 ++++++++++++++++++------- test/mutation/mutants.json | 35 +++- test/unit/backfill-incremental.test.ts | 132 ++++++++++++--- test/unit/backfill.test.ts | 11 +- 4 files changed, 304 insertions(+), 93 deletions(-) diff --git a/src/backfill.ts b/src/backfill.ts index 4996616..d4cdd13 100644 --- a/src/backfill.ts +++ b/src/backfill.ts @@ -540,10 +540,12 @@ interface BackfillState { complete: boolean; /** * The first day the file was ASKED to start from: `--since`, or where the - * archive starts, whichever is later. Not the same as `start` on a plan - * short of the full archive, where the request is clamped up to the first - * day the key may read. It is what lets a nightly cron with a fixed - * `--since` before that day keep running. Absent from version-1 files. + * archive starts, whichever is later. `start` is the first day the file + * actually covers, after the key's floor; on a plan short of the full archive + * the two differ, and this is what lets a nightly cron with a fixed `--since` + * before that floor keep running. Absent from version-1 files, which are + * judged by `start` alone. The same field, meaning the same thing, as the + * Python SDK's. */ since?: string | null; /** @@ -551,7 +553,7 @@ interface BackfillState { * was appended after the checkpoint (a page cut off by an interruption) and is * truncated before a resume, so a resume never appends a day twice. */ - bytes?: number; + size?: number; } function readState(path: string): BackfillState | null { @@ -735,12 +737,14 @@ function refuse(outPath: string, why: string): number { * leave a gap the state file could not describe. Both are refused rather than * quietly ignored, which would hand back a file that is not what was asked for. * - * A `--since` inside the file is fine and common: a cron line with a fixed - * `--since` runs it every night, and one computed as "30 days ago" moves - * forward every night. "Before the file" is judged against what the file was - * ASKED to start from, not where it does start: on a plan short of the full - * archive the first request is clamped up to the key's first day, and a fixed - * `--since` before that day has to keep working the next night. + * THE SAME `--since` IS ALWAYS ACCEPTED, even when it is before the file's + * first day. On a plan short of the full archive, `--since 2025-01-01` starts + * the file at the first day the key can read, and a cron line repeating it every + * night must keep working. So a `--since` is judged against the first day + * WRITTEN, except that the one the file was asked to start from (`since` in the + * state) is always fine. A `--since` inside the file is fine and common too: one + * computed as "30 days ago" moves forward every night. Before the archive starts + * is the same as its start. The rule and the wording match the Python SDK. */ function rangeFits( outPath: string, @@ -752,12 +756,12 @@ function rangeFits( const since = opts.since != null ? later(opts.since, archiveFrom) : null; const until = opts.until ?? null; const fileStart = state.start; - const askedFrom = state.since ?? fileStart; - if (since !== null && askedFrom !== null && since < askedFrom) { + const asked = state.since ?? fileStart; + if (since !== null && fileStart !== null && since < fileStart && since !== asked) { return refuse( outPath, - `it was started from ${String(fileStart)}, and --since ${String(opts.since)} would ` + - `need days before that. Appending cannot add them`, + `it was started from ${fileStart}, and --since ${String(opts.since)} would need days ` + + `before that. Appending cannot add them`, ); } if (until !== null && fileStart !== null && until < fileStart) { @@ -794,10 +798,6 @@ export function decide( rmSync(statePath, { force: true }); } let state = opts.overwrite ? null : readState(statePath); - if (state !== null) { - const cut = cutToCheckpoint(outPath, state, opts.format); - if (cut !== null) return cut; - } const hasRows = (): boolean => existsSync(outPath) && statSync(outPath).size > 0; let fileHasRows = hasRows(); @@ -838,41 +838,102 @@ export function decide( ); return 1; } + if (state !== null) { + const checked = backToCheckpoint(outPath, statePath, state, opts.format); + if (typeof checked === 'number') return checked; + state = checked; + } if (state === null) return firstRun(outPath, opts, archiveFrom, end); return continueFile(outPath, state, opts, archiveFrom, end); } +/** The most park-local days one page of the park daily endpoint covers. */ +const PAGE_DAYS = 31; + /** - * Put the data file back to the size it had at its last checkpoint. + * Cut the file back to what the state vouches for: the checkpoint's `size`. + * + * THIS IS WHAT MAKES A RERUN IDEMPOTENT. The state used to be written only at + * the end of a run or on an error this code caught, so Ctrl-C, SIGTERM or a kill + * during a nightly extension left `complete: true` with the old end, and the + * rerun appended the same days again. The state is now written after every + * page with the file's size at that moment, so whatever is past that size was + * written after the last checkpoint (half a page, or a torn line) and is + * discarded here, then fetched again. * - * Rows are appended as they arrive and the checkpoint is written once a page is - * complete, so an interruption (Ctrl-C, SIGTERM, a kill, a crash) can leave - * part of a page after the last checkpoint. Resuming at the checkpoint's day - * fetches that page again, and appending it would duplicate every row already - * there. Truncating to the recorded size first makes the resume exact. Null to - * proceed, or an exit code when the file is SHORTER than recorded, which means - * something else changed it and nothing here can say which days it lost. + * A file SHORTER than its checkpoint was changed by something else, and + * appending would leave a hole the state claims is filled, so it is refused. + * + * A state with no `size` predates it. Its file is cut back by date instead, to + * the day it continues from, which reads the file once and then records a size. + * Returns the state to go on with, null when nothing is left in the file (a + * first run), or an exit code. The same rule, and the same words, as the + * Python SDK. */ -function cutToCheckpoint(outPath: string, state: BackfillState, format: string): number | null { - if (state.complete || typeof state.bytes !== 'number' || !existsSync(outPath)) return null; - if (stateMismatch(state, format) !== null) return null; - const size = statSync(outPath).size; - if (size === state.bytes) return null; - if (size < state.bytes) { +function backToCheckpoint( + outPath: string, + statePath: string, + state: BackfillState, + format: string, +): BackfillState | null | number { + const actual = statSync(outPath).size; + const size = state.size; + if (typeof size !== 'number') { + const keepThrough = legacyKeepThrough(state); + if (keepThrough !== null && trimAfter(outPath, format, keepThrough) === 0) { + rmSync(outPath, { force: true }); + rmSync(statePath, { force: true }); + return null; + } + const next: BackfillState = { ...state }; + if (keepThrough !== null && !state.complete) next.resumeFrom = nextDay(keepThrough); + next.size = statSync(outPath).size; + writeState(statePath, next); + return next; + } + if (actual < size) { + return refuse( + outPath, + `it is ${String(actual)} bytes, shorter than the ${String(size)} its state file ` + + `records, so something other than this command changed it`, + ); + } + if (actual > size) { process.stderr.write( - ` ${basename(outPath)} is shorter than when this command last recorded it ` + - `(${String(size)} bytes, not ${String(state.bytes)}), so rows it held are gone.\n` + - ` --overwrite download this park again\n` + - ` or move both files aside and run again\n`, + ` discarding the last ${String(actual - size)} bytes of ${basename(outPath)}: written ` + + `after the last checkpoint, and fetched again now\n`, ); - return 1; + truncateSync(outPath, size); } - truncateSync(outPath, state.bytes); - process.stderr.write( - ` ${basename(outPath)}: removed the rows written after the last checkpoint ` + - `(an interrupted page); they are fetched again\n`, - ); - return null; + if (size === 0) { + // Nothing survived the cut: this is a first run, from the range asked. + rmSync(statePath, { force: true }); + return null; + } + return state; +} + +/** + * For a state without `size`: the newest day its file can be trusted to hold. + * + * A finished file holds every day through `end`. An unfinished one holds every + * day before `resumeFrom`, the page boundary; rows from that day on were written + * after it, by a run that was then killed. Without a boundary there is only + * `lastDay`, the newest day ANY entity reached, and rows arrive entity by entity, + * so another entity may have stopped days earlier. The page that run was on + * began at most 30 days before `lastDay`, so that is where it is safe to go back + * to. Resuming at `lastDay` itself lost the later entities' days for good. + */ +function legacyKeepThrough(state: BackfillState): string | null { + if (state.complete) return state.end; + if (state.resumeFrom !== null) return daysBefore(state.resumeFrom, 1); + const floor = state.start !== null ? daysBefore(state.start, 1) : null; + if (state.lastDay !== null) { + // No earlier than the file's own first day: nothing before it exists. + const back = daysBefore(state.lastDay, PAGE_DAYS); + return floor === null || back >= floor ? back : floor; + } + return floor; } /** A park with no file yet: from `--since`, or wherever the archive starts. */ @@ -942,11 +1003,14 @@ function continueFile( const continueAt = state.resumeFrom ?? state.lastDay ?? state.start ?? archiveFrom; const refused = rangeFits(outPath, state, opts, archiveFrom, String(continueAt)); if (refused !== null) return refused; - // A resume at `lastDay` refetches a day the file already holds. Only a state - // with no byte count can get here (one written before checkpoints recorded - // it), so that day's rows are removed first and the resume stays exact. - if (state.resumeFrom === null && state.lastDay !== null && typeof state.bytes !== 'number') { - trimAfter(outPath, opts.format, daysBefore(state.lastDay, 1)); + if (continueAt !== null && continueAt > end && opts.until != null && end === opts.until) { + // Not the key's window closing: the file already holds every day this + // --until asks for, and carries on from after it on a run without one. + process.stderr.write( + ` nothing to fetch into ${basename(outPath)}: it already holds the days through ` + + `--until ${opts.until}, and continues from ${continueAt}\n`, + ); + return 0; } return { ...carried, start: continueAt, extending: false }; } @@ -994,15 +1058,29 @@ function upgradeV1( const upgraded: BackfillState = { ...state, stateVersion: STATE_VERSION }; if (state.end === null) return upgraded; const keepThrough = daysBefore(state.end, V1_UNSETTLED_DAYS); + const name = basename(outPath); if (state.start !== null && keepThrough < state.start) { + process.stderr.write( + ` ${name} was written by 8.3, and every day in it is within ${String(V1_UNSETTLED_DAYS)} ` + + `days of that run's end, so none of it is known to be final: downloading it again\n`, + ); rmSync(outPath, { force: true }); rmSync(statePath, { force: true }); return null; } const lastDay = state.lastDay; if (lastDay === null || lastDay > keepThrough) { + process.stderr.write( + ` ${name} was written by 8.3: removing its rows after ${keepThrough}, which may be ` + + `partial days, and fetching those days again\n`, + ); trimAfter(outPath, format, keepThrough); upgraded.lastDay = lastDay !== null ? keepThrough : null; + } else { + process.stderr.write( + ` ${name} was written by 8.3; its newest row is older than any partial day, so ` + + `only its state file is updated\n`, + ); } if (state.complete) { upgraded.end = keepThrough; @@ -1012,6 +1090,8 @@ function upgradeV1( upgraded.resumeFrom = nextDay(keepThrough); } } + // The size now, so the checkpoint cut does not read the file a second time. + upgraded.size = statSync(outPath).size; writeState(statePath, upgraded); return upgraded; } @@ -1060,11 +1140,16 @@ function csvCells(record: string): string[] { * written back, so quoting is untouched by construction. A CSV record ends at a * newline outside quotes; `csvLine` quotes every cell holding a CR or LF, so a * name with a line break in it stays one record. + * + * Returns how many rows were kept, counting any it could not read and not + * counting the CSV header. */ -export function trimAfter(outPath: string, format: string, keepThrough: string): void { +export function trimAfter(outPath: string, format: string, keepThrough: string): number { const scratch = `${outPath}.trimming`; const input = openSync(outPath, 'r'); const output = openSync(scratch, 'w'); + let kept = 0; + let done = false; try { const decoder = new StringDecoder('utf8'); const chunk = Buffer.alloc(1 << 20); @@ -1104,14 +1189,22 @@ export function trimAfter(outPath: string, format: string, keepThrough: string): if (format === 'csv' && c === '"') quoted = !quoted; else if (c === '\n' && !(format === 'csv' && quoted)) { const record = pending.slice(from, i + 1); - if (keep(record)) writeSync(output, record); + const wasHeader = format === 'csv' && header; + if (keep(record)) { + writeSync(output, record); + if (!wasHeader) kept += 1; + } from = i + 1; } } pending = pending.slice(from); scanned = pending.length; if (final && pending !== '') { - if (keep(pending)) writeSync(output, pending); + const wasHeader = format === 'csv' && header; + if (keep(pending)) { + writeSync(output, pending); + if (!wasHeader) kept += 1; + } pending = ''; } }; @@ -1124,11 +1217,15 @@ export function trimAfter(outPath: string, format: string, keepThrough: string): } pending += decoder.end(); drain(true); + done = true; } finally { closeSync(input); closeSync(output); + // The half-written copy is removed, and the original left as it was. + if (!done) rmSync(scratch, { force: true }); } renameSync(scratch, outPath); + return kept; } /** The minimum of a writable stream this module needs, so a test can stand in. */ @@ -1207,9 +1304,8 @@ export async function backfillPark(tp: ThemeParks, park: Park, opts: RunOptions) const owner = acquireLock(lockPath); if (owner !== null) { process.stderr.write( - ` another run is writing ${basename(outPath)}` + - `${owner > 0 ? ` (process ${String(owner)})` : ''}. Wait for it to finish, or ` + - `stop it, then run again\n`, + `${park.id}: another themeparks-backfill is writing this park into ${opts.outDir} ` + + `right now. Two at once would each append the same days; wait for it to finish\n`, ); return 1; } @@ -1314,8 +1410,8 @@ async function runPark( lastDay: lastDay ?? priorLastDay, resumeFrom, complete, - since: since ?? fileStart, - bytes: existsSync(outPath) ? statSync(outPath).size : 0, + since, + size: existsSync(outPath) ? statSync(outPath).size : 0, }); /** @@ -1623,8 +1719,10 @@ export async function main( // access exists is worse than either. if (apiKey == null && !listing) { process.stderr.write( - 'no API key: reading the 7 days anonymous access allows.\n' + - ' a free key reads 30 days, and the paid tiers reach further\n' + + 'no API key: reading the 7 days anonymous access allows. Only final days\n' + + ' are written, and the newest 2 to 3 are still being recorded, so that\n' + + ' is usually 4 or 5 days per park.\n' + + ' a free key reads 30 days, Pro 400, Business the whole archive\n' + ' set THEMEPARKS_API_KEY, or pass --api-key\n' + ' keys: https://www.themeparks.wiki/profile\n\n', ); @@ -1745,7 +1843,8 @@ export async function main( const anonymousNotice = (): void => { if (apiKey != null) return; process.stderr.write( - `\nthat was ANONYMOUS ACCESS: the last 7 days only.\n` + + `\nthat was ANONYMOUS ACCESS: the final days among the last 7 days only,\n` + + ` usually 4 or 5 per park.\n` + ` a free key reads 30 days, Pro 400, Business the whole archive\n` + ` set THEMEPARKS_API_KEY and run the same command again\n` + ` keys: https://www.themeparks.wiki/profile\n`, diff --git a/test/mutation/mutants.json b/test/mutation/mutants.json index 0c267b2..16ffe7f 100644 --- a/test/mutation/mutants.json +++ b/test/mutation/mutants.json @@ -179,8 +179,8 @@ { "name": "an earlier --since than the file is accepted", "file": "src/backfill.ts", - "find": "if (since !== null && askedFrom !== null && since < askedFrom) {", - "replace": "if (false) {", + "find": " if (since !== null && fileStart !== null && since < fileStart && since !== asked) {", + "replace": " if (false) {", "why": "Appending cannot add days before the file's first one, so accepting it hands back a file that is not what was asked for." }, { @@ -193,8 +193,8 @@ { "name": "a fixed --since before the key's floor is refused the next night", "file": "src/backfill.ts", - "find": "const askedFrom = state.since ?? fileStart;", - "replace": "const askedFrom = fileStart;", + "find": "since < fileStart && since !== asked) {", + "replace": "since < fileStart) {", "why": "On a plan short of the archive the file starts at the key's first day, so judging --since against that refused the same cron line the next night." }, { @@ -249,8 +249,8 @@ { "name": "an anonymous 8.3 file is trimmed to nothing", "file": "src/backfill.ts", - "find": "keepThrough < state.start) {", - "replace": "keepThrough < '') {", + "find": " if (state.start !== null && keepThrough < state.start) {", + "replace": " if (false) {", "why": "A file wholly inside the cut would be emptied and continued from a day the key can no longer read." }, { @@ -291,7 +291,7 @@ { "name": "a resume does not cut back to the checkpoint", "file": "src/backfill.ts", - "find": " truncateSync(outPath, state.bytes);", + "find": " truncateSync(outPath, size);", "replace": "", "why": "Rows written after the last checkpoint by an interrupted page were kept, and the resume appended the same page again." }, @@ -308,6 +308,27 @@ "find": " if (owner !== null) {", "replace": " if (false) {", "why": "Two runs appending to one file interleave rows and each records its own checkpoint." + }, + { + "name": "an 8.3 file starting on the cut is downloaded again", + "file": "src/backfill.ts", + "find": "keepThrough < state.start) {", + "replace": "keepThrough <= state.start) {", + "why": "A file that starts on the last kept day has a day that is final; restarting refetched the whole file." + }, + { + "name": "a run that adds no rows forgets the file's newest day", + "file": "src/backfill.ts", + "find": " lastDay: lastDay ?? priorLastDay,", + "replace": " lastDay,", + "why": "A seasonal park closed for the new days recorded lastDay null, so the state no longer said what the file holds." + }, + { + "name": "a sizeless state resumes at lastDay", + "file": "src/backfill.ts", + "find": " const back = daysBefore(state.lastDay, PAGE_DAYS);", + "replace": " const back = daysBefore(state.lastDay, 1);", + "why": "lastDay is the newest day ANY entity reached; resuming there lost the days later entities had not reached." } ] } diff --git a/test/unit/backfill-incremental.test.ts b/test/unit/backfill-incremental.test.ts index c34cacb..a239e7d 100644 --- a/test/unit/backfill-incremental.test.ts +++ b/test/unit/backfill-incremental.test.ts @@ -364,6 +364,27 @@ describe('a rerun adds new days', () => { expect(stderr()).toContain('(new days)'); }); + it('keeps the newest day in the file when a run adds no rows', async () => { + // A seasonal park closed for the new days: nothing is written, and the state + // must still say which day the file's newest row is. + const archive = new Archive(); + await run(archive); + archive.entities = []; + archive.advance(3); + expect(await run(archive)).toBe(0); + expect(state()).toMatchObject({ lastDay: '2026-09-26', end: '2026-09-29', complete: true }); + }); + + it('an --until the file already holds, on an unfinished file, says so', async () => { + const archive = new Archive(); + archive.budgetOnPage = 3; + expect(await run(archive)).toBe(EX_TEMPFAIL); + archive.budgetOnPage = null; + expect(await run(archive, 'ndjson', { until: '2026-07-01' })).toBe(0); + expect(stderr()).toContain('already holds the days through --until 2026-07-01'); + expect(stderr()).not.toContain('Your plan no longer reaches'); + }); + it('keeps the original start and moves the end', async () => { const archive = new Archive(); await run(archive); @@ -596,8 +617,36 @@ describe('--since and --until', () => { archive.floor = '2026-08-28'; await run(archive, 'ndjson', { since: '2025-01-01' }); expect(await run(archive, 'ndjson', { since: '2024-01-01' })).toBe(1); + // Names the day the file really starts on. + expect(stderr()).toContain( + 'it was started from 2026-08-28, and --since 2024-01-01 would need days before that', + ); expect(stderr()).toContain('--overwrite'); }); + + it('records start as the first day covered and since as what was asked', async () => { + // Agreed with the Python SDK: `start` is where the file really begins, + // after the key's floor; `since` is the start it was asked for. + const limited = new Archive('2021-07-03'); + limited.floor = '2026-08-28'; + await run(limited, 'ndjson', { since: '2025-01-01' }); + expect(state()).toMatchObject({ start: '2026-08-28', since: '2025-01-01' }); + await run(new Archive('2021-07-03'), 'csv'); + expect(state('csv')).toMatchObject({ start: '2021-07-03', since: '2021-07-03' }); + }); + + it('refuses a --since between the one asked for and the first day written', async () => { + // The same rule as the Python SDK: only the --since the file was asked to + // start from is accepted before its first day. Any other would ask for days + // the file does not hold and a rerun cannot add. + const limited = new Archive('2021-07-03'); + limited.floor = '2026-08-28'; + await run(limited, 'ndjson', { since: '2025-01-01' }); + expect(await run(limited, 'ndjson', { since: '2026-01-01' })).toBe(1); + expect(stderr()).toContain( + 'it was started from 2026-08-28, and --since 2026-01-01 would need days before that', + ); + }); }); // --------------------------------------------------------------------------- @@ -846,20 +895,42 @@ describe('a file 8.3 wrote is corrected once, not frozen', () => { expect(archive.calls).toEqual([['2026-07-01', '2026-09-26']]); }); - it('starts again an anonymous 8.3 file that lies wholly inside the cut', async () => { - // Anonymous access reads 7 days, so every row such a file holds is within - // seven days of its end. Trimming would leave it empty, continuing from a day - // the key can no longer read; it is fetched again instead, all final. - const anonymous = new Archive('2026-09-22'); - await writeAs83('ndjson', anonymous); - v1State('ndjson', { start: '2026-09-22' }); - // Two days later, the key's 7-day window has moved on past the file's start. - const archive = new Archive('2026-06-01', '2026-09-28', '2026-09-30', '2026-09-24'); + it.each(['ndjson', 'csv'] as const)( + 'starts again an anonymous 8.3 file that lies wholly inside the cut (%s)', + async (format) => { + // Anonymous access reads 7 days, so every row such a file holds is within + // seven days of its end. Trimming would leave it empty, continuing from a day + // the key can no longer read; it is fetched again instead, all final. + const anonymous = new Archive('2026-09-22'); + await writeAs83(format, anonymous); + v1State(format, { start: '2026-09-22' }); + // Two days later, the key's 7-day window has moved on past the file's start. + const archive = new Archive('2026-06-01', '2026-09-28', '2026-09-30', '2026-09-24'); + expect(await run(archive, format)).toBe(0); + expect(archive.calls.at(-1)).toEqual(['2026-09-24', '2026-09-28']); + expect(minDate(rows(format))).toBe('2026-09-24'); + expectEveryRowFinalAndUnique(rows(format)); + expect(state(format)).toMatchObject({ stateVersion: STATE_VERSION, end: '2026-09-28' }); + }, + ); + + it('trims, not restarts, a file whose first day is exactly the cut', async () => { + // The boundary: a file starting ON the last kept day keeps that day. + const archive = new Archive('2026-09-21'); + await writeAs83('ndjson', archive); + v1State('ndjson', { start: '2026-09-21' }); expect(await run(archive)).toBe(0); - expect(archive.calls.at(-1)).toEqual(['2026-09-24', '2026-09-28']); - expect(minDate(rows())).toBe('2026-09-24'); - expectEveryRowFinalAndUnique(rows()); - expect(state()).toMatchObject({ stateVersion: STATE_VERSION, end: '2026-09-28' }); + expect(archive.calls).toEqual([['2026-09-22', '2026-09-26']]); + expect(stderr()).toContain('removing its rows after 2026-09-21'); + contiguous(rows(), '2026-09-21', '2026-09-26'); + }); + + it('says what it did to an 8.3 file', async () => { + const archive = new Archive('2026-06-01', '2026-07-31', '2026-07-31'); + await writeAs83('ndjson', archive); + v1State('ndjson', { end: '2026-09-28', lastDay: '2026-07-31' }); + await run(archive); + expect(stderr()).toContain('only its state file is updated'); }); it('still refuses an old state file from the other SDK', async () => { @@ -941,10 +1012,12 @@ describe('a run killed part-way through', () => { const saved = state(format); // Pages one and two are checkpointed; page three was cut off. expect(saved).toMatchObject({ complete: false, resumeFrom: '2026-08-02' }); - expect(statSync(dataPath(format)).size).toBeGreaterThan(saved.bytes as number); + expect(statSync(dataPath(format)).size).toBeGreaterThan(saved.size as number); expect(await run(archive, format)).toBe(0); - expect(stderr()).toContain('removed the rows written after the last checkpoint'); + expect(stderr()).toMatch( + /discarding the last \d+ bytes of p\.\w+: written after the last checkpoint, and fetched again now/u, + ); expectEveryRowFinalAndUnique(rows(format)); contiguous(rows(format), '2026-06-01', '2026-09-26'); if (format === 'csv') { @@ -967,7 +1040,7 @@ describe('a run killed part-way through', () => { it('leaves a state to resume from when killed before its first row', async () => { const archive = new Archive(); await killMidPage(archive, 1, 0); - expect(state()).toMatchObject({ complete: false, resumeFrom: '2026-06-01', bytes: 0 }); + expect(state()).toMatchObject({ complete: false, resumeFrom: '2026-06-01', size: 0 }); expect(await run(archive)).toBe(0); contiguous(rows(), '2026-06-01', '2026-09-26'); }); @@ -984,16 +1057,27 @@ describe('a run killed part-way through', () => { it('refuses a file shorter than its checkpoint says', async () => { const archive = new Archive(); await killMidPage(archive, 3, 10); - truncateSync(dataPath(), (state().bytes as number) - 10); + truncateSync(dataPath(), (state().size as number) - 10); expect(await run(archive)).toBe(1); - expect(stderr()).toContain('shorter than when this command last recorded it'); + expect(stderr()).toContain( + 'its state file records, so something other than this command changed it', + ); }); - it('drops the day a state with no byte count would resume on, before refetching it', async () => { + it('cuts a sizeless state back a whole page before its last day, then resumes', async () => { // A state from before checkpoints recorded the file's size, interrupted in - // its first page: it resumes on `lastDay`, which the file already holds. + // its first page. `lastDay` is the newest day ANY entity reached; rows go + // entity by entity, so another entity may have stopped earlier. Resuming at + // `lastDay` lost those days; going back a page (31 days) cannot. const archive = new Archive('2026-06-01', '2026-07-10', '2026-07-10'); await writeAs83('ndjson', archive); + // The second entity got no further than 2026-06-20 before the kill. + const text = readFileSync(dataPath(), 'utf8') + .split('\n') + .filter((l) => l === '' || !(l.includes('"ent-b"') && l.includes('"date":"2026-06-2'))) + .filter((l) => l === '' || !(l.includes('"ent-b"') && /"date":"2026-0(6-3|7)/u.test(l))) + .join('\n'); + writeFileSync(dataPath(), text); writeFileSync( statePathFor(dir, 'p', 'ndjson'), JSON.stringify({ @@ -1011,9 +1095,11 @@ describe('a run killed part-way through', () => { ); const later = new Archive(); expect(await run(later)).toBe(0); - expect(later.calls).toEqual([['2026-07-10', '2026-09-26']]); + expect(later.calls).toEqual([['2026-06-10', '2026-09-26']]); expectEveryRowFinalAndUnique(rows()); contiguous(rows(), '2026-06-01', '2026-09-26'); + // Both entities have every day, including the ones ent-b had not reached. + expect(rows().filter((r) => r.entityId === 'ent-b')).toHaveLength(118); }); it('writes the state file whole, leaving no scratch file behind', async () => { @@ -1032,7 +1118,9 @@ describe('one run per output file', () => { await first.hung; const second = new Archive(); expect(await run(second)).toBe(1); - expect(stderr()).toContain('another run is writing p.ndjson'); + expect(stderr()).toContain( + `p: another themeparks-backfill is writing this park into ${dir} right now`, + ); expect(second.calls).toEqual([]); releaseLocks(); }); diff --git a/test/unit/backfill.test.ts b/test/unit/backfill.test.ts index 1b4e273..2ded2c3 100644 --- a/test/unit/backfill.test.ts +++ b/test/unit/backfill.test.ts @@ -512,9 +512,10 @@ describe('decide', () => { }); }); - it('falls back to lastDay for a state file written before resumeFrom existed', () => { - // One duplicated day beats starting from the top and appending a second - // copy of the whole archive. + it('goes back a page from lastDay for a state with no resumeFrom and no size', () => { + // Rows arrive entity by entity, so `lastDay` (the newest day ANY entity + // reached) can be past days another entity never got to. A page is at most + // 31 days, so going back that far loses nothing and duplicates nothing. writeFileSync(out, '{"a":1}\n'); stateFile(dir, { start: '2021-07-03', @@ -523,7 +524,7 @@ describe('decide', () => { complete: false, }); expect(decide(out, state, opts(), '2021-07-03', '2026-09-23')).toMatchObject({ - start: '2026-08-30', + start: '2026-07-31', hasRows: true, }); }); @@ -1377,6 +1378,8 @@ describe('an anonymous run says so when it finishes', () => { expect(text).toContain('ANONYMOUS ACCESS'); // Both ends: before, so it can be acted on, and after, so it is read. expect(text.split('7 days').length - 1).toBeGreaterThanOrEqual(2); + // And does not promise seven days of rows: the newest days are held back. + expect(text).toContain('usually 4 or 5'); expect(text.trimEnd().endsWith('keys: https://www.themeparks.wiki/profile')).toBe(true); }); From afc318ec7a4330c9396399c8c220fb81388328ee Mon Sep 17 00:00:00 2001 From: Jamie Holding Date: Tue, 29 Sep 2026 09:09:44 +0100 Subject: [PATCH 7/7] docs: recorded rather than immutable; fetching a range again; stopping safely A day through recordedTo is one the archive has recorded and is fetched once; the archive can occasionally re-record a past day after a feed repair, so the README now says how to fetch a range again. Documents that stopping a run is safe at any point, what `opening.degraded` means, and, in the CHANGELOG, that this is 8.4.0 with `HistorySpan` gaining a required field. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 54 +++++++++++++++++++++++++++++++++++----- README.md | 40 +++++++++++++++++++++++------ examples/backfill.mjs | 4 +-- src/backfill.ts | 15 ++++++++--- src/ergonomic/history.ts | 9 ++++--- 5 files changed, 100 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7a9c7a..be0fcd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +To be released as **8.4.0**, a minor: the additions below are backwards +compatible at runtime. One is not quite so at the type level: `HistorySpan` +gains a required field, `finalThrough`, so code that builds a `HistorySpan` +object by hand (a test double, say) needs to add it. Code that only reads the +spans `span()` returns is unaffected. + ### Added - **`themeparks-backfill --since YYYY-MM-DD` and `--until YYYY-MM-DD`.** There @@ -29,11 +35,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 what it always did; it now also has `opening`, an object of `HistoryOpening` keyed by entity id, covering every entity in the response, including one that did not change all day. It is readable once the response has arrived: iterate - first, or `await changes.load()`. Either way it is one request. The - `HistoryChanges` and `HistoryOpening` types are exported. - -- **`HistorySpan.finalThrough`**: the newest day whose daily row will not change - again, the earlier of `recordedTo` and `retrievableThrough`. + first, or `await changes.load()`. Either way it is one request. It is not + enumerable, so spreading or logging the result is safe, and after a failed + request it says so. The `HistoryChanges` and `HistoryOpening` types are + exported, and the README explains `opening.degraded`. + +- **`HistorySpan.finalThrough`**: the newest day the archive has recorded that + the key may read, the earlier of `recordedTo` and `retrievableThrough`: the + place to stop if you fetch each day once. + +- **An interrupted `themeparks-backfill` never appends a day twice.** The state + file is written before the first request and after every page, atomically, + with the size of the data file at that moment; the next run first cuts the + file back to that size. Ctrl-C, SIGTERM (now exit 143, as well as Ctrl-C's 130) and a kill at any point cost at most the page in flight. Two runs on the + same park and `--out` at once are refused. + +- **`days()` awaits `onPage`.** A hook that returns a promise holds the next page + until it settles, so a checkpoint written there is on disk before the next + request. The hook's type is widened to return `unknown`, so existing callbacks + still compile. ### Fixed @@ -51,7 +71,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 so far, and the archive records days 2 to 3 behind live data, so the last few days of every file were still changing when they were written. A run now ends at `finalThrough`, says so when it holds days back, and the next run adds them - once they are final. Every row in the file is one that will not change. + once they are final. Each day is fetched once, as the archive recorded it; the + README says how to fetch a range again if the archive later re-records it. **Files written by 8.3.x are corrected once.** Their state file does not say which of their newest days were final, so the first run of this version @@ -75,6 +96,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 it a second time under a second header, exit 0. It is refused now, the same way. +- **An interrupted nightly extension appended the same days twice.** The state + was written only at the end of a run or on an error the command caught, so + Ctrl-C, SIGTERM or a kill during an extension left it saying finished through + the old day, and the rerun appended those days again. See the checkpointing + above. + +- **A run that stopped part-way through its first page lost days.** It resumed + from the newest day any entity had reached; rows arrive entity by entity, so + the entities behind it lost the days in between. Checkpoints make this + impossible for new files, and an older state with only that day goes back a + whole page (31 days) instead. + +- **The state recorded the start asked for, not the first day written.** `start` + is now the first day the file covers, after the key's window, and a new + `since` field keeps the start that was asked for, so the same `--since` keeps + working and a different one before the file is refused. + +- **The anonymous-access notice promised 7 days.** Only final days are written, + so an anonymous run writes usually 4 or 5 days per park, and the notice now + says so. + - **A state file whose data file had been deleted was continued**, producing a file that started part-way through its range and was then recorded as complete. The park is downloaded again from the start instead. diff --git a/README.md b/README.md index 63fd743..98c4be7 100644 --- a/README.md +++ b/README.md @@ -265,8 +265,9 @@ Three things to know before polling these: fails fast by default rather than looking hung. A REST 429, which asks for seconds, is still ridden out. - **Today is not final.** The default cache leaves `changes` and `daily` - uncached and keeps `coverage` for an hour. A completed day never changes, so - cache it yourself for as long as you like. + uncached and keeps `coverage` for an hour. A day on or before `recordedTo` + has been recorded, so cache it yourself for as long as you like; the archive + only re-records a past day in the rare case of a feed repair. `tp.raw.getEntityHistory(id, query)`, `getEntityHistoryDaily(id, query)` and `getEntityHistoryCoverage(id)` are the underlying calls. @@ -312,9 +313,10 @@ ends in 403s. **If you write each day once, end at `finalThrough`.** `retrievableThrough` is usually today, and today's row is the day so far. Recent days can still change too, because the archive records days 2 to 3 behind live data. `finalThrough` is -the earlier of `recordedTo` and `retrievableThrough`: the newest day whose row -will not change again. Store through that, and fetch the days after it on your -next run. +the earlier of `recordedTo` and `retrievableThrough`: the newest day the archive +has recorded. Store through that, and fetch the days after it on your next run. +The archive can occasionally re-record a past day, for example after a park's +feed is repaired, so fetch a range again if you need to pick that up. **`days()` yields, it does not collect.** Nothing accumulates, so the only thing that grows is whatever you write the rows to. @@ -359,7 +361,12 @@ every entity in the response, including one that did not change all day, and `opening.observedAt` says when that state was last seen, which can be long before the range for a ride whose feed stopped. `opening` is readable once the response has arrived: iterate first, or `await changes.load()`. Either way it is one -request. +request. It is not enumerable, so spreading or logging the result is safe. + +An opening with `degraded: true` is incomplete: the server could not look far +enough back for this response, and `degradedReason` says why (`timeout`, +`error` or `capacity`). A field it holds may be missing, so ask again in a +minute for the full opening before rebuilding a day from it. ### Or skip the code: there is a command @@ -391,8 +398,25 @@ alerting and the same command continues where it stopped. `--help` has the rest. **Run it again to update.** A second run of a finished park fetches only the days that have become final since the last one and appends them, so a nightly cron keeps the file current. Only final days are written: a run ends at the newest day -the archive has finished recording and says so when it holds newer days back, so -no row in the file changes later. +the archive has finished recording and says so when it holds newer days back. +Each day is fetched once, as the archive recorded it. + +**Fetching a range again.** The archive can occasionally re-record past days, +for example when a park's feed is repaired. A file never rewrites rows it +already holds, so to pick up a correction, download the affected days into a +separate directory and replace those `(entityId, date)` rows where you load the +data, or start the file again: + +```bash +npx themeparks-backfill "Epcot" --since 2026-06-01 --until 2026-06-30 --out ./refetch +npx themeparks-backfill "Epcot" --overwrite # or: the whole file again +``` + +**Stopping a run at any point is safe.** The state file is written after every +page with the size of the file at that moment, and the next run first cuts off +anything written after it, so no day is ever appended twice. Ctrl-C and SIGTERM +exit 130 and 143; a killed run needs nothing either. Two runs on the same park +and `--out` at once are refused. **`--since` and `--until`** (`YYYY-MM-DD`, both inclusive) pick the days, instead of everything your plan reaches. `--since` applies when a file is started: a diff --git a/examples/backfill.mjs b/examples/backfill.mjs index 8c7c514..b0f04fb 100644 --- a/examples/backfill.mjs +++ b/examples/backfill.mjs @@ -18,8 +18,8 @@ * (retrievableThrough) and what the archive holds (recordedTo). Asking past * the entitlement is how a long backfill ends in 403s, and the days past * recordedTo are not final: today's row is the day so far, and the archive - * records days 2 to 3 behind live data. Stopping there means no row this - * writes will change later. + * records days 2 to 3 behind live data. Stopping there means each day is + * written once, as the archive recorded it. * * 3. It checkpoints. The history budget is hourly, so a spent one can be most * of an hour from resetting. The SDK raises BudgetExhaustedError rather diff --git a/src/backfill.ts b/src/backfill.ts index d4cdd13..498fe70 100644 --- a/src/backfill.ts +++ b/src/backfill.ts @@ -36,8 +36,10 @@ * 5. It writes FINAL days only. Today's row is the day so far, and the archive * records days 2 to 3 behind live data, so the newest days the API serves can * still change. The run ends at `span().finalThrough`, the newest day the - * archive holds, and the next run carries on from the day after. Every row in - * the file is one that will not change, so a nightly run only ever appends. + * archive has recorded, and the next run carries on from the day after. Each + * day is fetched once, as the archive recorded it, so a nightly run only ever + * appends. (The archive can re-record a past day after a feed repair; the + * README says how to fetch a range again.) */ // Several internals are exported for tests. They are not in the package's public // surface: `bin` points at this file and `src/index.ts` does not re-export it, so @@ -1617,7 +1619,14 @@ examples: only final days are written. Today's row is the day so far, and the archive records days 2 to 3 behind live data, so the newest days can still change. A run ends at the newest final day and the next run carries on from the day after, so -the file only ever grows and no row in it changes later. +the file only ever grows, and each day is fetched once, as the archive recorded +it. To pick up a day the archive re-recorded later, fetch that range into a +different --out (--since and --until) and replace those rows where you load them. + +stopping a run is safe at any point: the state file is written after every page +with the size of the file, and the next run cuts off anything written after it, +so no day is appended twice. Ctrl-C and SIGTERM exit 130 and 143. Two runs on +the same park and --out at once are refused. --since applies when a file is started. A later run continues that file forward and accepts the same --since, or a later one. One earlier than the file's first diff --git a/src/ergonomic/history.ts b/src/ergonomic/history.ts index 459aca2..c0e6949 100644 --- a/src/ergonomic/history.ts +++ b/src/ergonomic/history.ts @@ -118,13 +118,16 @@ export interface HistorySpan { */ retrievableThrough: string | null; /** - * The newest day whose daily rows will not change again, or null. + * The newest day the archive has recorded that this key may read, or null: + * the place to stop if you fetch each day once. * * `retrievableThrough` is usually today, and today's row is the day so far. * Recent days can still change after that too: the archive records days 2 to * 3 behind live data. `recordedTo` is the newest day the archive holds, so a - * day on or before it is final. Store those, and ask for anything later again - * once `finalThrough` has moved past it. + * day on or before it has been recorded. Store those, and ask for anything + * later once `finalThrough` has moved past it. The archive can occasionally + * re-record a past day, for example after a park's feed is repaired; fetch + * that range again if you need the correction. * * The earlier of `recordedTo` and `retrievableThrough`, because a key may be * entitled to fewer days than the archive holds. Null when either is unknown.