From d12e5af6180bf4f122160e437817e1ff4d42301c Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 9 Sep 2026 11:36:20 -0700 Subject: [PATCH 1/8] Add: plan retention recovery from restart evidence Retention publication refuses every retained stage as recovery-required, and nothing yet decides what a retained stage means. This adds the storage-independent half of that decision. assess_root_stage, assess_manifest_stage, and assess_head_stage classify each fixed stage as absent, complete, truncated, or corrupt, using the decoders' own truncation variants so a crash mid-write and a complete-looking record that fails a checksum are told apart. RetentionRecoveryEvidence binds those assessments to the observed current state and to whether each pool already holds the entry a complete stage names. plan_retention_recovery is pure over that evidence and applies the documented classification: a truncated stage with no later-ordered effect is discarded; a complete root or manifest stage is linked into its pool and retained as a recovery-protected orphan; a complete head over linked stages is finalized and both stages removed; stages the published head already names are cleaned up; everything else is a typed RetentionRecoveryRefusal before any effect. Eleven laws over the golden version-two records cover every crash prefix the recovery page names, including the successor case against a published generation. No I/O happens here; the storage port and executor follow. Refs #19 --- CHANGELOG.md | 10 + docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/retention.rs | 18 + .../retention/filesystem_retention_current.rs | 17 + src/adapters/retention/recovery_evidence.rs | 96 +++++ src/adapters/retention/recovery_plan.rs | 70 ++++ src/adapters/retention/recovery_planner.rs | 283 ++++++++++++++ .../retention/recovery_planner_tests.rs | 356 ++++++++++++++++++ src/adapters/retention/recovery_refusal.rs | 174 +++++++++ .../retention/recovery_stage_assessment.rs | 92 +++++ src/lib.rs | 20 +- 11 files changed, 1130 insertions(+), 8 deletions(-) create mode 100644 src/adapters/retention/recovery_evidence.rs create mode 100644 src/adapters/retention/recovery_plan.rs create mode 100644 src/adapters/retention/recovery_planner.rs create mode 100644 src/adapters/retention/recovery_planner_tests.rs create mode 100644 src/adapters/retention/recovery_refusal.rs create mode 100644 src/adapters/retention/recovery_stage_assessment.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 080ae98..3561f7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,16 @@ after its public API and format compatibility policies are established. ### Added +- Storage-independent retention recovery planning: `assess_root_stage`, + `assess_manifest_stage`, and `assess_head_stage` classify each fixed stage + as absent, complete, truncated, or corrupt through the decoders' own + truncation laws; `plan_retention_recovery` turns that evidence, the observed + current state, and pool-entry observations into an ordered + `RetentionRecoveryPlan` (discard a pre-effect truncated stage, link and + protect complete orphans, finalize a complete head over linked stages, clean + up stages the published head already names) or a typed + `RetentionRecoveryRefusal`. Planning performs no I/O; storage execution is + the next step. - `FilesystemRetentionPublicationAuthority` executes the 17 ordered retention publication phases against a completely migrated version-2 root. It stages `root.next`, `manifest.next`, and `head.next` exclusively, verifies device diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index ae8100b..ff8ae2f 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -15,7 +15,7 @@ case is not evidence. | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; writer-locked initial filesystem publication in `filesystem_retention_storage_tests`; observed-head successor publication, exact predecessor binding, and absent-head refusal in `filesystem_retention_successor_tests`; the store's catalog head must name the closure's catalog generation and digest before any forward write in `filesystem_retention_catalog_tests`; a head whose predecessor disagrees with its manifest refuses in `filesystem_retention_current_tests`; a successor reopens and decodes the manifest-selected predecessor root and refuses an absent or changed one in `filesystem_retention_expectation_tests` | Implemented | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; production 17-phase forward filesystem execution, exclusive staging, byte-equal inode-substitution refusal, and retained-stage recovery refusal in `filesystem_retention_storage_tests`; orphan namespace directories count against the 4,096 ceiling and refuse a new namespace before any stage is written in `filesystem_retention_capacity_tests`; crash injection remains | In progress in #19 | -| `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | recovery-required refusals before any mutation in `filesystem_retention_expectation_tests`: an absent head over populated pools, a non-initial head prepared against an absent head, an orphan directory for a namespace expected absent, and an absent directory for a namespace expected current; debug and release crash matrix remains; replaced protocol directories, an absent or changed head-selected catalog, an over-full census, zero-generation pool names, and a stage retained by a failed write refuse in `filesystem_retention_*_tests` | In progress in #19 | +| `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | recovery-required refusals before any mutation in `filesystem_retention_expectation_tests`: an absent head over populated pools, a non-initial head prepared against an absent head, an orphan directory for a namespace expected absent, and an absent directory for a namespace expected current; debug and release crash matrix remains; replaced protocol directories, an absent or changed head-selected catalog, an over-full census, zero-generation pool names, and a stage retained by a failed write refuse in `filesystem_retention_*_tests`; storage-independent classification of every fixed-stage crash prefix (discard, link and protect, finalize, clean up, or typed refusal) in `recovery_planner_tests` | In progress in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | | `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests`; committed retry reopens the head-selected manifest entry and root pool bytes, refusing absent, changed, or corrupt evidence in `filesystem_retention_current_tests`; every refusal is a typed `RetentionCurrentStateRefusal` source, with superseded, committed-root-absent, committed-root-changed, and head-absent-with-artifacts pinned by downcast | Implemented | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | model-based and source-architecture tests | Planned in #19 | diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 334f250..1e6b308 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -91,6 +91,13 @@ mod transition_preflight_error; mod transition_readiness; mod verified_closure; +mod recovery_evidence; +mod recovery_plan; +mod recovery_planner; +#[cfg(test)] +mod recovery_planner_tests; +mod recovery_refusal; +mod recovery_stage_assessment; pub use admitted_manifest::AdmittedRetentionManifest; pub use admitted_root::AdmittedRetentionRoot; pub use canonical_head::CanonicalRetentionHead; @@ -118,6 +125,17 @@ pub use publication_preparation::prepare_retention_publication; pub use publication_preparation_error::RetentionPublicationPreparationError; pub use publication_receipt::RetentionPublicationReceipt; pub use publication_storage::RetentionPublicationStorage; +pub use recovery_evidence::{ + RetentionPoolEntryObservation, RetentionPoolObservations, RetentionRecoveryEvidence, + RetentionStageAssessments, +}; +pub use recovery_plan::{RetentionRecoveryOutcome, RetentionRecoveryPlan, RetentionRecoveryStep}; +pub use recovery_planner::plan_retention_recovery; +pub use recovery_refusal::{RetentionFixedStage, RetentionPool, RetentionRecoveryRefusal}; +pub use recovery_stage_assessment::{ + RetentionHeadStageAssessment, RetentionManifestStageAssessment, RetentionRootStageAssessment, + RetentionStageAssessment, assess_head_stage, assess_manifest_stage, assess_root_stage, +}; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; pub use transition_disposition::RetentionTransitionDisposition; diff --git a/src/adapters/retention/filesystem_retention_current.rs b/src/adapters/retention/filesystem_retention_current.rs index fd22ae2..1179320 100644 --- a/src/adapters/retention/filesystem_retention_current.rs +++ b/src/adapters/retention/filesystem_retention_current.rs @@ -57,6 +57,23 @@ impl ObservedRetentionState { } } +#[cfg(test)] +impl ObservedRetentionState { + /// Builds the observed state from exact head and manifest bytes. + pub(super) fn for_tests(head: &[u8], manifest: &[u8]) -> io::Result { + let decoded = ChecksummedRetentionHead::decode(head) + .map_err(|source| RetentionCurrentStateRefusal::HeadRefused { source }.into_io())?; + let admitted = AdmittedRetentionManifest::decode(manifest) + .map_err(|source| RetentionCurrentStateRefusal::ManifestRefused { source }.into_io())?; + Ok(Self { + head: Box::from(head), + manifest: Box::from(manifest), + decoded_head: *decoded.head(), + decoded_manifest: admitted.manifest().clone(), + }) + } +} + /// The verified relationship between one preparation and the observed state. #[derive(Clone, Copy)] pub(super) enum ObservedDisposition<'state> { diff --git a/src/adapters/retention/recovery_evidence.rs b/src/adapters/retention/recovery_evidence.rs new file mode 100644 index 0000000..de12663 --- /dev/null +++ b/src/adapters/retention/recovery_evidence.rs @@ -0,0 +1,96 @@ +//! This module owns the complete evidence retention recovery plans from. + +use super::{ + ObservedRetentionState, RetentionHeadStageAssessment, RetentionManifestStageAssessment, + RetentionRootStageAssessment, +}; + +/// Whether an immutable pool already holds the entry a complete stage names. +/// +/// The observation is meaningful only for a `Complete` stage: a truncated or +/// corrupt stage names no canonical entry, and the adapter reports `Absent`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionPoolEntryObservation { + /// No entry exists under the canonical name. + Absent, + /// The entry exists with exactly the stage's bytes. + Identical, + /// The entry exists with other bytes or another kind. + Different, +} + +/// The three fixed-stage assessments read at restart. +#[derive(Debug)] +pub struct RetentionStageAssessments<'bytes> { + /// Assessment of `retention/root.next`. + pub root: RetentionRootStageAssessment<'bytes>, + /// Assessment of `retention/manifest.next`. + pub manifest: RetentionManifestStageAssessment<'bytes>, + /// Assessment of `retention/head.next`. + pub head: RetentionHeadStageAssessment<'bytes>, +} + +/// Whether each immutable pool holds the entry its complete stage names. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RetentionPoolObservations { + /// The root pool entry the complete root stage names. + pub root: RetentionPoolEntryObservation, + /// The manifest pool entry the complete manifest stage names. + pub manifest: RetentionPoolEntryObservation, +} + +/// Everything restart observed before planning retention recovery. +/// +/// The evidence is read under exclusive writer authority and performs no +/// mutation; planning over it is pure. +#[derive(Debug)] +pub struct RetentionRecoveryEvidence<'bytes, 'state> { + current: Option<&'state ObservedRetentionState>, + stages: RetentionStageAssessments<'bytes>, + pools: RetentionPoolObservations, +} + +impl<'bytes, 'state> RetentionRecoveryEvidence<'bytes, 'state> { + /// Binds the observed current state, the three stage assessments, and the + /// pool observations for the entries the complete stages name. + #[must_use] + pub const fn new( + current: Option<&'state ObservedRetentionState>, + stages: RetentionStageAssessments<'bytes>, + pools: RetentionPoolObservations, + ) -> Self { + Self { + current, + stages, + pools, + } + } + + pub(super) fn into_parts( + self, + ) -> ( + Option<&'state ObservedRetentionState>, + RetentionStageAssessments<'bytes>, + RetentionPoolObservations, + ) { + (self.current, self.stages, self.pools) + } + + /// The published head and manifest, or `None` when no head is published. + #[must_use] + pub const fn current(&self) -> Option<&'state ObservedRetentionState> { + self.current + } + + /// The three stage assessments. + #[must_use] + pub const fn stages(&self) -> &RetentionStageAssessments<'bytes> { + &self.stages + } + + /// The pool observations for the entries the complete stages name. + #[must_use] + pub const fn pools(&self) -> RetentionPoolObservations { + self.pools + } +} diff --git a/src/adapters/retention/recovery_plan.rs b/src/adapters/retention/recovery_plan.rs new file mode 100644 index 0000000..6de4670 --- /dev/null +++ b/src/adapters/retention/recovery_plan.rs @@ -0,0 +1,70 @@ +//! This module owns the typed retention recovery plan and its outcome. + +/// One ordered recovery effect. Each maps to exactly one storage capability. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionRecoveryStep { + /// Remove a truncated `head.next` whose replacement never happened. + DiscardHeadStage, + /// Remove a truncated `manifest.next` that was never linked. + DiscardManifestStage, + /// Remove a truncated `root.next` that was never linked. + DiscardRootStage, + /// Admit the namespace directory and link the complete root stage into it. + LinkRoot, + /// Link the complete manifest stage into the manifest pool. + LinkManifest, + /// Replace `retention/HEAD` with the complete head stage and synchronize. + FinalizeHead, + /// Remove the retained root stage after its pool link is proven. + RemoveRootStage, + /// Remove the retained manifest stage after its pool link is proven. + RemoveManifestStage, +} + +/// The state recovery leaves once every step has executed. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionRecoveryOutcome { + /// No stage remains; forward publication may proceed. + Clean, + /// The staged generation is (or was already) the published head and its + /// stages are removed; forward publication may proceed. + Committed, + /// Complete stages remain linked and retained as valid orphans. They are + /// recovery-protected until explicit disposition; forward publication + /// refuses meanwhile. + Protected { + /// `root.next` remains retained. + root_stage: bool, + /// `manifest.next` remains retained. + manifest_stage: bool, + }, +} + +/// The ordered effects recovery must execute and the state they produce. +#[derive(Clone, Debug, Eq, PartialEq)] +#[must_use] +pub struct RetentionRecoveryPlan { + steps: Vec, + outcome: RetentionRecoveryOutcome, +} + +impl RetentionRecoveryPlan { + pub(super) const fn new( + steps: Vec, + outcome: RetentionRecoveryOutcome, + ) -> Self { + Self { steps, outcome } + } + + /// The effects in execution order; empty when nothing must change. + #[must_use] + pub fn steps(&self) -> &[RetentionRecoveryStep] { + &self.steps + } + + /// The state the store is in after every step executes. + #[must_use] + pub const fn outcome(&self) -> RetentionRecoveryOutcome { + self.outcome + } +} diff --git a/src/adapters/retention/recovery_planner.rs b/src/adapters/retention/recovery_planner.rs new file mode 100644 index 0000000..63e0d78 --- /dev/null +++ b/src/adapters/retention/recovery_planner.rs @@ -0,0 +1,283 @@ +//! This module owns pure planning of retention recovery from restart evidence. + +use super::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, ChecksummedRetentionHead, + ObservedRetentionState, RetentionFixedStage, RetentionPool, + RetentionPoolEntryObservation as Pool, RetentionPoolObservations, RetentionRecoveryEvidence, + RetentionRecoveryOutcome, RetentionRecoveryPlan, RetentionRecoveryRefusal as Refusal, + RetentionRecoveryStep as Step, RetentionStageAssessment as Stage, +}; +use crate::{LivenessGeneration, RootGeneration}; + +type Current<'state> = Option<&'state ObservedRetentionState>; + +/// Plans retention recovery from complete restart evidence. +/// +/// The call performs no I/O. It applies the documented classification: a +/// truncated stage with no later-ordered effect is discarded; a complete +/// root or manifest stage is linked into its pool and retained as a +/// recovery-protected orphan; a complete head stage naming the staged +/// manifest is finalized and both retained stages are removed; a staged +/// generation the published head already names is cleaned up. Any other +/// combination is unrecoverable ambiguity and refuses before any effect. +/// +/// # Errors +/// +/// Returns [`RetentionRecoveryRefusal`](super::RetentionRecoveryRefusal) naming +/// the exact ambiguity. +pub fn plan_retention_recovery( + evidence: RetentionRecoveryEvidence<'_, '_>, +) -> Result { + let head_present = evidence.stages().head.is_present(); + let manifest_present = evidence.stages().manifest.is_present(); + let (current, stages, pools) = evidence.into_parts(); + let mut steps = Vec::new(); + let head = match stages.head { + Stage::Absent => None, + Stage::Truncated { .. } => { + steps.push(Step::DiscardHeadStage); + None + } + Stage::Corrupt(source) => return Err(Refusal::corrupt_head(source)), + Stage::Complete(head) => Some(head), + }; + let manifest = match stages.manifest { + Stage::Absent => None, + Stage::Truncated { .. } => { + if head_present || pools.manifest != Pool::Absent { + return Err(Refusal::TruncatedStageWithLaterEffect { + stage: RetentionFixedStage::Manifest, + }); + } + steps.push(Step::DiscardManifestStage); + None + } + Stage::Corrupt(source) => return Err(Refusal::corrupt_manifest(source)), + Stage::Complete(manifest) => Some(manifest), + }; + let root = match stages.root { + Stage::Absent => None, + Stage::Truncated { .. } => { + if head_present || manifest_present || pools.root != Pool::Absent { + return Err(Refusal::TruncatedStageWithLaterEffect { + stage: RetentionFixedStage::Root, + }); + } + steps.push(Step::DiscardRootStage); + None + } + Stage::Corrupt(source) => return Err(Refusal::corrupt_root(source)), + Stage::Complete(root) => Some(root), + }; + if root.is_some() && pools.root == Pool::Different { + return Err(Refusal::PoolEntryDiffers { + pool: RetentionPool::Roots, + }); + } + if manifest.is_some() && pools.manifest == Pool::Different { + return Err(Refusal::PoolEntryDiffers { + pool: RetentionPool::Manifests, + }); + } + match (head, manifest, root) { + (Some(head), Some(manifest), Some(root)) => finalize_head( + current, + CompleteStages { + head: &head, + manifest: &manifest, + root: &root, + }, + pools, + steps, + ), + (Some(_), _, _) => Err(Refusal::HeadStageWithoutManifestStage), + (None, Some(manifest), root) => { + plan_manifest(current, &manifest, root.as_ref(), pools, steps) + } + (None, None, Some(root)) => plan_root(current, &root, pools.root, steps), + (None, None, None) => Ok(RetentionRecoveryPlan::new( + steps, + RetentionRecoveryOutcome::Clean, + )), + } +} + +/// The three complete stages a head finalization is planned from. +#[derive(Clone, Copy)] +struct CompleteStages<'a, 'bytes> { + head: &'a ChecksummedRetentionHead<'bytes>, + manifest: &'a AdmittedRetentionManifest<'bytes>, + root: &'a AdmittedRetentionRoot<'bytes>, +} + +fn finalize_head( + current: Current<'_>, + stages: CompleteStages<'_, '_>, + pools: RetentionPoolObservations, + mut steps: Vec, +) -> Result { + let CompleteStages { + head, + manifest, + root, + } = stages; + let head = head.head(); + if head.manifest_digest() != manifest.digest() + || head.generation() != manifest.manifest().generation() + { + return Err(Refusal::HeadStageNamesOtherManifest); + } + if !manifest_names_root(manifest, root) { + return Err(Refusal::ManifestStageNamesOtherRoot); + } + if pools.root != Pool::Identical { + return Err(Refusal::RootNotLinkedBeforeHead); + } + if pools.manifest != Pool::Identical { + return Err(Refusal::ManifestNotLinkedBeforeHead); + } + if !is_committed(current, manifest) && !manifest_succeeds(current, manifest) { + return Err(Refusal::HeadPredecessorMismatch); + } + steps.extend([ + Step::FinalizeHead, + Step::RemoveRootStage, + Step::RemoveManifestStage, + ]); + Ok(RetentionRecoveryPlan::new( + steps, + RetentionRecoveryOutcome::Committed, + )) +} + +fn plan_manifest( + current: Current<'_>, + manifest: &AdmittedRetentionManifest<'_>, + root: Option<&AdmittedRetentionRoot<'_>>, + pools: RetentionPoolObservations, + mut steps: Vec, +) -> Result { + if is_committed(current, manifest) { + if let Some(root) = root { + if !manifest_names_root(manifest, root) { + return Err(Refusal::ManifestStageNamesOtherRoot); + } + if pools.root != Pool::Identical { + return Err(Refusal::RootNotLinkedBeforeHead); + } + steps.push(Step::RemoveRootStage); + } + steps.push(Step::RemoveManifestStage); + return Ok(RetentionRecoveryPlan::new( + steps, + RetentionRecoveryOutcome::Committed, + )); + } + let root = root.ok_or(Refusal::ManifestStageWithoutRootStage)?; + if !manifest_names_root(manifest, root) { + return Err(Refusal::ManifestStageNamesOtherRoot); + } + if !manifest_succeeds(current, manifest) { + return Err(Refusal::ManifestNotSuccessor); + } + if !root_succeeds(current, root) { + return Err(Refusal::RootNotSuccessor); + } + if pools.root == Pool::Absent { + steps.push(Step::LinkRoot); + } + if pools.manifest == Pool::Absent { + steps.push(Step::LinkManifest); + } + Ok(RetentionRecoveryPlan::new( + steps, + RetentionRecoveryOutcome::Protected { + root_stage: true, + manifest_stage: true, + }, + )) +} + +fn plan_root( + current: Current<'_>, + root: &AdmittedRetentionRoot<'_>, + root_pool: Pool, + mut steps: Vec, +) -> Result { + if !root_succeeds(current, root) { + return Err(Refusal::RootNotSuccessor); + } + if root_pool == Pool::Absent { + steps.push(Step::LinkRoot); + } + Ok(RetentionRecoveryPlan::new( + steps, + RetentionRecoveryOutcome::Protected { + root_stage: true, + manifest_stage: false, + }, + )) +} + +/// Whether the published head already names the staged manifest. +fn is_committed(current: Current<'_>, manifest: &AdmittedRetentionManifest<'_>) -> bool { + current.is_some_and(|current| { + current.head().manifest_digest() == manifest.digest() + && current.head().generation() == manifest.manifest().generation() + }) +} + +fn manifest_names_root( + manifest: &AdmittedRetentionManifest<'_>, + root: &AdmittedRetentionRoot<'_>, +) -> bool { + let namespace = root.root().namespace().digest(); + let entries = manifest.manifest().entries(); + entries + .binary_search_by_key(&namespace, |entry| entry.namespace()) + .ok() + .and_then(|index| entries.get(index)) + .is_some_and(|entry| { + entry.root_generation() == root.root().generation() + && entry.root_digest() == root.digest() + }) +} + +fn manifest_succeeds(current: Current<'_>, manifest: &AdmittedRetentionManifest<'_>) -> bool { + let manifest = manifest.manifest(); + current.map_or_else( + || manifest.predecessor().is_none() && manifest.generation() == LivenessGeneration::INITIAL, + |current| { + manifest.predecessor() == Some(current.head().manifest_digest()) + && current + .head() + .generation() + .successor() + .is_ok_and(|successor| successor == manifest.generation()) + }, + ) +} + +fn root_succeeds(current: Current<'_>, root: &AdmittedRetentionRoot<'_>) -> bool { + let namespace = root.root().namespace().digest(); + let entry = current.and_then(|current| { + let entries = current.manifest().entries(); + entries + .binary_search_by_key(&namespace, |entry| entry.namespace()) + .ok() + .and_then(|index| entries.get(index).copied()) + }); + entry.map_or_else( + || { + root.root().predecessor().is_none() + && root.root().generation() == RootGeneration::INITIAL + }, + |entry| { + root.root().predecessor() == Some(entry.root_digest()) + && entry + .root_generation() + .successor() + .is_ok_and(|successor| successor == root.root().generation()) + }, + ) +} diff --git a/src/adapters/retention/recovery_planner_tests.rs b/src/adapters/retention/recovery_planner_tests.rs new file mode 100644 index 0000000..2dba6d2 --- /dev/null +++ b/src/adapters/retention/recovery_planner_tests.rs @@ -0,0 +1,356 @@ +//! Retention recovery planning laws over the golden version-two records. + +use std::error::Error; + +use super::filesystem_retention_test_fixture::{ + HEAD_HEX, MANIFEST_HEX, ROOT_HEX, fixture, initial_preparation, successor_preparation, +}; +use super::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, ObservedRetentionState, RetentionFixedStage, + RetentionPool, RetentionPoolEntryObservation as Pool, RetentionPoolObservations, + RetentionRecoveryEvidence, RetentionRecoveryOutcome as Outcome, RetentionRecoveryRefusal, + RetentionRecoveryStep as Step, RetentionStageAssessments, assess_head_stage, + assess_manifest_stage, assess_root_stage, plan_retention_recovery, +}; + +struct Records { + root: Vec, + manifest: Vec, + head: Vec, +} + +fn generation_one() -> Result> { + Ok(Records { + root: fixture(ROOT_HEX)?, + manifest: fixture(MANIFEST_HEX)?, + head: fixture(HEAD_HEX)?, + }) +} + +type StageBytes<'b> = (Option<&'b [u8]>, Option<&'b [u8]>, Option<&'b [u8]>); + +fn evidence<'b, 's>( + current: Option<&'s ObservedRetentionState>, + (root, manifest, head): StageBytes<'b>, + (root_pool, manifest_pool): (Pool, Pool), +) -> RetentionRecoveryEvidence<'b, 's> { + RetentionRecoveryEvidence::new( + current, + RetentionStageAssessments { + root: assess_root_stage(root), + manifest: assess_manifest_stage(manifest), + head: assess_head_stage(head), + }, + RetentionPoolObservations { + root: root_pool, + manifest: manifest_pool, + }, + ) +} + +fn corrupt(bytes: &[u8]) -> Vec { + let mut bytes = bytes.to_vec(); + if let Some(last) = bytes.last_mut() { + *last ^= 0x01; + } + bytes +} + +#[test] +fn a_clean_store_needs_nothing() -> Result<(), Box> { + let plan = plan_retention_recovery(evidence( + None, + (None, None, None), + (Pool::Absent, Pool::Absent), + ))?; + assert!(plan.steps().is_empty()); + assert_eq!(plan.outcome(), Outcome::Clean); + Ok(()) +} + +#[test] +fn a_truncated_root_stage_with_no_later_effect_is_discarded() -> Result<(), Box> { + let records = generation_one()?; + let partial = records + .root + .get(..100) + .ok_or("root fixture shorter than 100 bytes")?; + let plan = plan_retention_recovery(evidence( + None, + (Some(partial), None, None), + (Pool::Absent, Pool::Absent), + ))?; + assert_eq!(plan.steps(), [Step::DiscardRootStage]); + assert_eq!(plan.outcome(), Outcome::Clean); + Ok(()) +} + +#[test] +fn a_truncated_stage_with_a_later_effect_refuses() -> Result<(), Box> { + let records = generation_one()?; + let partial = records + .manifest + .get(..100) + .ok_or("manifest fixture shorter than 100 bytes")?; + let error = plan_retention_recovery(evidence( + None, + (Some(&records.root), Some(partial), None), + (Pool::Identical, Pool::Identical), + )) + .err() + .ok_or("a truncated manifest with a pool link was discarded")?; + assert!(matches!( + error, + RetentionRecoveryRefusal::TruncatedStageWithLaterEffect { + stage: RetentionFixedStage::Manifest + } + )); + Ok(()) +} + +#[test] +fn a_corrupt_stage_refuses_with_its_decode_error() -> Result<(), Box> { + let records = generation_one()?; + let corrupt_root = corrupt(&records.root); + let error = plan_retention_recovery(evidence( + None, + (Some(&corrupt_root), None, None), + (Pool::Absent, Pool::Absent), + )) + .err() + .ok_or("a corrupt root stage was planned")?; + assert!(matches!( + error, + RetentionRecoveryRefusal::StageCorrupt { + stage: RetentionFixedStage::Root, + .. + } + )); + assert!(error.source().is_some()); + Ok(()) +} + +#[test] +fn a_complete_root_stage_is_linked_and_protected() -> Result<(), Box> { + let records = generation_one()?; + let unlinked = plan_retention_recovery(evidence( + None, + (Some(&records.root), None, None), + (Pool::Absent, Pool::Absent), + ))?; + let linked = plan_retention_recovery(evidence( + None, + (Some(&records.root), None, None), + (Pool::Identical, Pool::Absent), + ))?; + let differs = plan_retention_recovery(evidence( + None, + (Some(&records.root), None, None), + (Pool::Different, Pool::Absent), + )) + .err() + .ok_or("a conflicting root pool entry was planned over")?; + assert_eq!(unlinked.steps(), [Step::LinkRoot]); + assert!(linked.steps().is_empty()); + for plan in [&unlinked, &linked] { + assert_eq!( + plan.outcome(), + Outcome::Protected { + root_stage: true, + manifest_stage: false + } + ); + } + assert!(matches!( + differs, + RetentionRecoveryRefusal::PoolEntryDiffers { + pool: RetentionPool::Roots + } + )); + Ok(()) +} + +#[test] +fn complete_root_and_manifest_stages_are_linked_and_protected() -> Result<(), Box> { + let records = generation_one()?; + let plan = plan_retention_recovery(evidence( + None, + (Some(&records.root), Some(&records.manifest), None), + (Pool::Absent, Pool::Absent), + ))?; + assert_eq!(plan.steps(), [Step::LinkRoot, Step::LinkManifest]); + assert_eq!( + plan.outcome(), + Outcome::Protected { + root_stage: true, + manifest_stage: true + } + ); + Ok(()) +} + +#[test] +fn a_complete_head_stage_over_linked_stages_is_finalized() -> Result<(), Box> { + let records = generation_one()?; + let plan = plan_retention_recovery(evidence( + None, + ( + Some(&records.root), + Some(&records.manifest), + Some(&records.head), + ), + (Pool::Identical, Pool::Identical), + ))?; + assert_eq!( + plan.steps(), + [ + Step::FinalizeHead, + Step::RemoveRootStage, + Step::RemoveManifestStage + ] + ); + assert_eq!(plan.outcome(), Outcome::Committed); + let unlinked = plan_retention_recovery(evidence( + None, + ( + Some(&records.root), + Some(&records.manifest), + Some(&records.head), + ), + (Pool::Absent, Pool::Identical), + )) + .err() + .ok_or("a head stage over an unlinked root was finalized")?; + assert!(matches!( + unlinked, + RetentionRecoveryRefusal::RootNotLinkedBeforeHead + )); + Ok(()) +} + +#[test] +fn a_truncated_head_stage_is_discarded_and_the_orphans_stay_protected() -> Result<(), Box> +{ + let records = generation_one()?; + let partial = records + .head + .get(..40) + .ok_or("head fixture shorter than 40 bytes")?; + let plan = plan_retention_recovery(evidence( + None, + (Some(&records.root), Some(&records.manifest), Some(partial)), + (Pool::Identical, Pool::Identical), + ))?; + assert_eq!(plan.steps(), [Step::DiscardHeadStage]); + assert_eq!( + plan.outcome(), + Outcome::Protected { + root_stage: true, + manifest_stage: true + } + ); + Ok(()) +} + +#[test] +fn a_head_stage_without_the_staged_manifest_refuses() -> Result<(), Box> { + let records = generation_one()?; + let error = plan_retention_recovery(evidence( + None, + (Some(&records.root), None, Some(&records.head)), + (Pool::Identical, Pool::Absent), + )) + .err() + .ok_or("a head stage without a manifest stage was finalized")?; + assert!(matches!( + error, + RetentionRecoveryRefusal::HeadStageWithoutManifestStage + )); + Ok(()) +} + +#[test] +fn stages_the_published_head_already_names_are_cleaned_up() -> Result<(), Box> { + let records = generation_one()?; + let current = ObservedRetentionState::for_tests(&records.head, &records.manifest)?; + let both = plan_retention_recovery(evidence( + Some(¤t), + (Some(&records.root), Some(&records.manifest), None), + (Pool::Identical, Pool::Identical), + ))?; + let manifest_only = plan_retention_recovery(evidence( + Some(¤t), + (None, Some(&records.manifest), None), + (Pool::Absent, Pool::Identical), + ))?; + let head_too = plan_retention_recovery(evidence( + Some(¤t), + ( + Some(&records.root), + Some(&records.manifest), + Some(&records.head), + ), + (Pool::Identical, Pool::Identical), + ))?; + assert_eq!( + both.steps(), + [Step::RemoveRootStage, Step::RemoveManifestStage] + ); + assert_eq!(manifest_only.steps(), [Step::RemoveManifestStage]); + assert_eq!( + head_too.steps(), + [ + Step::FinalizeHead, + Step::RemoveRootStage, + Step::RemoveManifestStage + ] + ); + for plan in [&both, &manifest_only, &head_too] { + assert_eq!(plan.outcome(), Outcome::Committed); + } + Ok(()) +} + +#[test] +fn a_successor_generation_is_planned_against_the_published_state() -> Result<(), Box> { + let records = generation_one()?; + let current = ObservedRetentionState::for_tests(&records.head, &records.manifest)?; + let current_root = AdmittedRetentionRoot::decode(&records.root)?; + let current_manifest = AdmittedRetentionManifest::decode(&records.manifest)?; + let candidate = super::filesystem_retention_test_fixture::successor_root(¤t_root)?; + let preparation = successor_preparation(¤t_root, ¤t_manifest, candidate.encoded())?; + let publication = preparation + .publication() + .ok_or("successor preparation carries no publication")?; + let manifest = publication.manifest().encoded().to_vec(); + let head = publication.head().encoded().to_vec(); + + let finalize = plan_retention_recovery(evidence( + Some(¤t), + (Some(candidate.encoded()), Some(&manifest), Some(&head)), + (Pool::Identical, Pool::Identical), + ))?; + let stale = plan_retention_recovery(evidence( + None, + (Some(candidate.encoded()), Some(&manifest), None), + (Pool::Absent, Pool::Absent), + )) + .err() + .ok_or("a successor manifest over an absent head was planned")?; + + assert_eq!( + finalize.steps(), + [ + Step::FinalizeHead, + Step::RemoveRootStage, + Step::RemoveManifestStage + ] + ); + assert_eq!(finalize.outcome(), Outcome::Committed); + assert!(matches!( + stale, + RetentionRecoveryRefusal::ManifestNotSuccessor + )); + drop(initial_preparation(&records.root)?); + Ok(()) +} diff --git a/src/adapters/retention/recovery_refusal.rs b/src/adapters/retention/recovery_refusal.rs new file mode 100644 index 0000000..449b394 --- /dev/null +++ b/src/adapters/retention/recovery_refusal.rs @@ -0,0 +1,174 @@ +//! This module owns the typed refusals of retention recovery planning. + +use std::error::Error; +use std::fmt; + +use super::{RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionRootDecodeError}; + +/// One of the three fixed retention stage names. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionFixedStage { + /// `retention/root.next`. + Root, + /// `retention/manifest.next`. + Manifest, + /// `retention/head.next`. + Head, +} + +/// One of the two immutable retention pools. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionPool { + /// `retention/roots/`. + Roots, + /// `retention/manifests`. + Manifests, +} + +/// Why the observed stages are unrecoverable ambiguity rather than a plan. +/// +/// Every variant leaves the store untouched: recovery refuses before any +/// effect, and the evidence stays for explicit disposition. +#[derive(Debug)] +#[non_exhaustive] +pub enum RetentionRecoveryRefusal { + /// A stage is complete enough to judge and fails a canonical law. + StageCorrupt { + /// The stage that failed. + stage: RetentionFixedStage, + /// The exact decode refusal. + source: Box, + }, + /// A truncated stage has a later-ordered effect, so it is not pre-effect. + TruncatedStageWithLaterEffect { + /// The truncated stage. + stage: RetentionFixedStage, + }, + /// A complete stage names a pool entry that exists with other bytes. + PoolEntryDiffers { + /// The pool holding the conflicting entry. + pool: RetentionPool, + }, + /// A complete head stage exists without a complete manifest stage. + HeadStageWithoutManifestStage, + /// The head stage names a manifest other than the staged one. + HeadStageNamesOtherManifest, + /// The head stage's predecessor is not the published manifest. + HeadPredecessorMismatch, + /// The head stage exists but the staged manifest was never linked. + ManifestNotLinkedBeforeHead, + /// The head stage exists but the staged root was never linked. + RootNotLinkedBeforeHead, + /// A complete manifest stage exists without the root stage it introduces + /// and is not the published manifest. + ManifestStageWithoutRootStage, + /// The manifest stage does not select the staged root. + ManifestStageNamesOtherRoot, + /// The manifest stage is not the exact successor of the published manifest. + ManifestNotSuccessor, + /// The root stage is not the exact successor of the namespace's current root. + RootNotSuccessor, +} + +impl RetentionRecoveryRefusal { + pub(super) fn corrupt_root(source: RetentionRootDecodeError) -> Self { + Self::StageCorrupt { + stage: RetentionFixedStage::Root, + source: Box::new(source), + } + } + + pub(super) fn corrupt_manifest(source: RetentionManifestDecodeError) -> Self { + Self::StageCorrupt { + stage: RetentionFixedStage::Manifest, + source: Box::new(source), + } + } + + pub(super) fn corrupt_head(source: RetentionHeadDecodeError) -> Self { + Self::StageCorrupt { + stage: RetentionFixedStage::Head, + source: Box::new(source), + } + } +} + +impl fmt::Display for RetentionFixedStage { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Root => "root.next", + Self::Manifest => "manifest.next", + Self::Head => "head.next", + }) + } +} + +impl fmt::Display for RetentionPool { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Roots => "retention/roots", + Self::Manifests => "retention/manifests", + }) + } +} + +impl fmt::Display for RetentionRecoveryRefusal { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::StageCorrupt { stage, .. } => { + write!(formatter, "retention stage {stage} is corrupt") + } + Self::TruncatedStageWithLaterEffect { stage } => write!( + formatter, + "truncated retention stage {stage} has a later-ordered effect" + ), + Self::PoolEntryDiffers { pool } => { + write!( + formatter, + "{pool} holds a different entry under the staged name" + ) + } + other => formatter.write_str(other.message()), + } + } +} + +impl RetentionRecoveryRefusal { + const fn message(&self) -> &'static str { + match self { + Self::HeadStageWithoutManifestStage => { + "head.next exists without a complete manifest.next" + } + Self::HeadStageNamesOtherManifest => { + "head.next names a manifest other than manifest.next" + } + Self::HeadPredecessorMismatch => "head.next does not succeed the published manifest", + Self::ManifestNotLinkedBeforeHead => { + "head.next exists but manifest.next was never linked" + } + Self::RootNotLinkedBeforeHead => "head.next exists but root.next was never linked", + Self::ManifestStageWithoutRootStage => { + "manifest.next exists without root.next and is not the published manifest" + } + Self::ManifestStageNamesOtherRoot => "manifest.next does not select root.next", + Self::ManifestNotSuccessor => { + "manifest.next is not the successor of the published manifest" + } + Self::RootNotSuccessor => { + "root.next is not the successor of its namespace's current root" + } + Self::StageCorrupt { .. } + | Self::TruncatedStageWithLaterEffect { .. } + | Self::PoolEntryDiffers { .. } => "retention recovery refused", + } + } +} + +impl Error for RetentionRecoveryRefusal { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::StageCorrupt { source, .. } => Some(source.as_ref()), + _ => None, + } + } +} diff --git a/src/adapters/retention/recovery_stage_assessment.rs b/src/adapters/retention/recovery_stage_assessment.rs new file mode 100644 index 0000000..3fb7d29 --- /dev/null +++ b/src/adapters/retention/recovery_stage_assessment.rs @@ -0,0 +1,92 @@ +//! This module owns restart assessment of the three fixed retention stages. + +use super::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, ChecksummedRetentionHead, + RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionRootDecodeError, +}; + +/// One fixed retention stage as assessed from its exact bytes at restart. +/// +/// `Truncated` means the bytes end before the boundary the record's own +/// framing declares, which is the shape a crash during the stage write leaves +/// behind. Every other decode failure is `Corrupt`: a complete-looking record +/// that fails a checksum, digest, or semantic law is unrecoverable ambiguity, +/// never an incomplete write. +#[derive(Debug)] +pub enum RetentionStageAssessment { + /// No entry exists under the stage name. + Absent, + /// The bytes decode as one canonical record. + Complete(Record), + /// The bytes end before the declared record boundary. + Truncated { + /// The length the framing declares. + expected: usize, + /// The length that was present. + observed: usize, + }, + /// The bytes are complete enough to judge and fail a canonical law. + Corrupt(Error), +} + +/// Assessment of `retention/root.next`. +pub type RetentionRootStageAssessment<'bytes> = + RetentionStageAssessment, RetentionRootDecodeError>; +/// Assessment of `retention/manifest.next`. +pub type RetentionManifestStageAssessment<'bytes> = + RetentionStageAssessment, RetentionManifestDecodeError>; +/// Assessment of `retention/head.next`. +pub type RetentionHeadStageAssessment<'bytes> = + RetentionStageAssessment, RetentionHeadDecodeError>; + +impl RetentionStageAssessment { + /// Returns whether an entry exists under the stage name. + #[must_use] + pub const fn is_present(&self) -> bool { + !matches!(self, Self::Absent) + } +} + +/// Assesses the bytes found under `retention/root.next`, if any. +#[must_use] +pub fn assess_root_stage(bytes: Option<&[u8]>) -> RetentionRootStageAssessment<'_> { + match bytes.map(AdmittedRetentionRoot::decode) { + None => RetentionStageAssessment::Absent, + Some(Ok(root)) => RetentionStageAssessment::Complete(root), + Some(Err(RetentionRootDecodeError::Truncated { expected, observed })) => { + RetentionStageAssessment::Truncated { expected, observed } + } + Some(Err(source)) => RetentionStageAssessment::Corrupt(source), + } +} + +/// Assesses the bytes found under `retention/manifest.next`, if any. +#[must_use] +pub fn assess_manifest_stage(bytes: Option<&[u8]>) -> RetentionManifestStageAssessment<'_> { + match bytes.map(AdmittedRetentionManifest::decode) { + None => RetentionStageAssessment::Absent, + Some(Ok(manifest)) => RetentionStageAssessment::Complete(manifest), + Some(Err(RetentionManifestDecodeError::Truncated { expected, observed })) => { + RetentionStageAssessment::Truncated { expected, observed } + } + Some(Err(source)) => RetentionStageAssessment::Corrupt(source), + } +} + +/// Assesses the bytes found under `retention/head.next`, if any. +/// +/// The head is one fixed 144-byte record, so fewer bytes are a truncation and +/// more bytes are corruption. +#[must_use] +pub fn assess_head_stage(bytes: Option<&[u8]>) -> RetentionHeadStageAssessment<'_> { + match bytes.map(ChecksummedRetentionHead::decode) { + None => RetentionStageAssessment::Absent, + Some(Ok(head)) => RetentionStageAssessment::Complete(head), + Some(Err(RetentionHeadDecodeError::WrongLength { expected, observed })) + if observed < expected => + { + RetentionStageAssessment::Truncated { expected, observed } + } + Some(Err(source)) => RetentionStageAssessment::Corrupt(source), + } +} diff --git a/src/lib.rs b/src/lib.rs index 6611c89..f07545d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -137,15 +137,21 @@ pub use adapters::{ CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, FilesystemRetentionAuthorityError, FilesystemRetentionPublicationAuthority, ObservedRetentionState, PreparedRetentionPublication, RetentionAuthorityDirectory, - RetentionClosureVerificationError, RetentionCurrentStateRefusal, RetentionHeadDecodeError, - RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionNamespaceAdmission, + RetentionClosureVerificationError, RetentionCurrentStateRefusal, RetentionFixedStage, + RetentionHeadDecodeError, RetentionHeadStageAssessment, RetentionManifestDecodeError, + RetentionManifestEncodeError, RetentionManifestStageAssessment, RetentionNamespaceAdmission, + RetentionPool, RetentionPoolEntryObservation, RetentionPoolObservations, RetentionPublicationError, RetentionPublicationOutcome, RetentionPublicationPhase, RetentionPublicationPreparation, RetentionPublicationPreparationError, - RetentionPublicationReceipt, RetentionPublicationStorage, RetentionRootDecodeError, - RetentionRootEncodeError, RetentionTransitionDisposition, RetentionTransitionError, - RetentionTransitionPreflight, RetentionTransitionPreflightError, RetentionTransitionReadiness, - VerifiedRetentionClosure, execute_retention_publication, plan_retention_transition, - preflight_retention_transition, prepare_retention_publication, verify_retention_closure, + RetentionPublicationReceipt, RetentionPublicationStorage, RetentionRecoveryEvidence, + RetentionRecoveryOutcome, RetentionRecoveryPlan, RetentionRecoveryRefusal, + RetentionRecoveryStep, RetentionRootDecodeError, RetentionRootEncodeError, + RetentionRootStageAssessment, RetentionStageAssessment, RetentionStageAssessments, + RetentionTransitionDisposition, RetentionTransitionError, RetentionTransitionPreflight, + RetentionTransitionPreflightError, RetentionTransitionReadiness, VerifiedRetentionClosure, + assess_head_stage, assess_manifest_stage, assess_root_stage, execute_retention_publication, + plan_retention_recovery, plan_retention_transition, preflight_retention_transition, + prepare_retention_publication, verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, From 48c999880208775a6df50248cd7fe2e24aa6ddaf Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 9 Sep 2026 12:04:50 -0700 Subject: [PATCH 2/8] Add: execute a retention recovery plan through a blocking storage port RetentionRecoveryStorage names one durable capability per recovery step: discard a truncated stage, link a complete root or manifest stage into its pool, finalize the head, and remove a retained stage after its link is proven. Each capability owns its complete effect and the synchronization that makes it durable, so an implementation cannot report a step done before its evidence would survive process death. execute_retention_recovery runs a plan in order, calling exactly one capability per step, and stops at the first refusal with the refused step, the completed prefix, and the storage's error as source; the caller re-observes and re-plans rather than continuing from stale evidence. The receipt records the executed steps and the plan's outcome. Three laws against a recording fake storage pin the mapping, the empty plan, and the refusal prefix. Refs #19 --- CHANGELOG.md | 6 +- src/adapters/retention.rs | 8 ++ src/adapters/retention/recovery_execution.rs | 112 +++++++++++++++++ .../retention/recovery_execution_tests.rs | 113 ++++++++++++++++++ src/adapters/retention/recovery_storage.rs | 72 +++++++++++ src/lib.rs | 11 +- 6 files changed, 315 insertions(+), 7 deletions(-) create mode 100644 src/adapters/retention/recovery_execution.rs create mode 100644 src/adapters/retention/recovery_execution_tests.rs create mode 100644 src/adapters/retention/recovery_storage.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3561f7a..3f8865c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,10 @@ after its public API and format compatibility policies are established. `RetentionRecoveryPlan` (discard a pre-effect truncated stage, link and protect complete orphans, finalize a complete head over linked stages, clean up stages the published head already names) or a typed - `RetentionRecoveryRefusal`. Planning performs no I/O; storage execution is - the next step. + `RetentionRecoveryRefusal`. `RetentionRecoveryStorage` names one blocking + capability per step and `execute_retention_recovery` runs a plan in order, + stopping at the first refused step with the completed prefix named in + `RetentionRecoveryError`; the filesystem implementation is the next step. - `FilesystemRetentionPublicationAuthority` executes the 17 ordered retention publication phases against a completely migrated version-2 root. It stages `root.next`, `manifest.next`, and `head.next` exclusively, verifies device diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 1e6b308..5a75baf 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -92,12 +92,16 @@ mod transition_readiness; mod verified_closure; mod recovery_evidence; +mod recovery_execution; +#[cfg(test)] +mod recovery_execution_tests; mod recovery_plan; mod recovery_planner; #[cfg(test)] mod recovery_planner_tests; mod recovery_refusal; mod recovery_stage_assessment; +mod recovery_storage; pub use admitted_manifest::AdmittedRetentionManifest; pub use admitted_root::AdmittedRetentionRoot; pub use canonical_head::CanonicalRetentionHead; @@ -129,6 +133,9 @@ pub use recovery_evidence::{ RetentionPoolEntryObservation, RetentionPoolObservations, RetentionRecoveryEvidence, RetentionStageAssessments, }; +pub use recovery_execution::{ + RetentionRecoveryError, RetentionRecoveryReceipt, execute_retention_recovery, +}; pub use recovery_plan::{RetentionRecoveryOutcome, RetentionRecoveryPlan, RetentionRecoveryStep}; pub use recovery_planner::plan_retention_recovery; pub use recovery_refusal::{RetentionFixedStage, RetentionPool, RetentionRecoveryRefusal}; @@ -136,6 +143,7 @@ pub use recovery_stage_assessment::{ RetentionHeadStageAssessment, RetentionManifestStageAssessment, RetentionRootStageAssessment, RetentionStageAssessment, assess_head_stage, assess_manifest_stage, assess_root_stage, }; +pub use recovery_storage::RetentionRecoveryStorage; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; pub use transition_disposition::RetentionTransitionDisposition; diff --git a/src/adapters/retention/recovery_execution.rs b/src/adapters/retention/recovery_execution.rs new file mode 100644 index 0000000..813217a --- /dev/null +++ b/src/adapters/retention/recovery_execution.rs @@ -0,0 +1,112 @@ +//! This module owns ordered execution of one retention recovery plan. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{ + RetentionRecoveryOutcome, RetentionRecoveryPlan, RetentionRecoveryStep, + RetentionRecoveryStorage, +}; + +/// The complete record of one executed retention recovery plan. +#[derive(Clone, Debug, Eq, PartialEq)] +#[must_use] +pub struct RetentionRecoveryReceipt { + executed: Vec, + outcome: RetentionRecoveryOutcome, +} + +impl RetentionRecoveryReceipt { + /// Every step that executed, in order. + #[must_use] + pub fn executed(&self) -> &[RetentionRecoveryStep] { + &self.executed + } + + /// The state the store is in now. + #[must_use] + pub const fn outcome(&self) -> RetentionRecoveryOutcome { + self.outcome + } +} + +/// One refused recovery step and the steps that completed before it. +#[derive(Debug)] +pub struct RetentionRecoveryError { + step: RetentionRecoveryStep, + executed: Vec, + source: io::Error, +} + +impl RetentionRecoveryError { + /// The step that refused. + #[must_use] + pub const fn step(&self) -> RetentionRecoveryStep { + self.step + } + + /// Every step that completed before the refusal, in order. + #[must_use] + pub fn executed(&self) -> &[RetentionRecoveryStep] { + &self.executed + } +} + +impl fmt::Display for RetentionRecoveryError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "retention recovery step {:?} refused after {} completed step(s)", + self.step, + self.executed.len() + ) + } +} + +impl Error for RetentionRecoveryError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + Some(&self.source) + } +} + +/// Executes `plan` against `storage` in order, stopping at the first refusal. +/// +/// Each step calls exactly one storage capability. A refused step leaves the +/// completed steps' effects in place, names the step, and returns; the caller +/// re-observes and re-plans rather than continuing from stale evidence. +/// +/// # Errors +/// +/// Returns [`RetentionRecoveryError`] with the refused step, the completed +/// steps, and the storage's own error as source. +pub fn execute_retention_recovery( + storage: &mut S, + plan: &RetentionRecoveryPlan, +) -> Result { + let mut executed = Vec::with_capacity(plan.steps().len()); + for &step in plan.steps() { + let result = match step { + RetentionRecoveryStep::DiscardHeadStage => storage.discard_head_stage(), + RetentionRecoveryStep::DiscardManifestStage => storage.discard_manifest_stage(), + RetentionRecoveryStep::DiscardRootStage => storage.discard_root_stage(), + RetentionRecoveryStep::LinkRoot => storage.link_root(), + RetentionRecoveryStep::LinkManifest => storage.link_manifest(), + RetentionRecoveryStep::FinalizeHead => storage.finalize_head(), + RetentionRecoveryStep::RemoveRootStage => storage.remove_root_stage(), + RetentionRecoveryStep::RemoveManifestStage => storage.remove_manifest_stage(), + }; + if let Err(source) = result { + return Err(RetentionRecoveryError { + step, + executed, + source, + }); + } + executed.push(step); + } + Ok(RetentionRecoveryReceipt { + executed, + outcome: plan.outcome(), + }) +} diff --git a/src/adapters/retention/recovery_execution_tests.rs b/src/adapters/retention/recovery_execution_tests.rs new file mode 100644 index 0000000..d68ac80 --- /dev/null +++ b/src/adapters/retention/recovery_execution_tests.rs @@ -0,0 +1,113 @@ +//! Retention recovery execution laws against a recording fake storage. + +use std::error::Error; +use std::io; + +use super::{ + RetentionRecoveryError, RetentionRecoveryOutcome, RetentionRecoveryPlan, + RetentionRecoveryStep as Step, RetentionRecoveryStorage, execute_retention_recovery, +}; + +#[derive(Default)] +struct Recording { + calls: Vec, + refuse_at: Option, +} + +impl Recording { + fn record(&mut self, step: Step) -> io::Result<()> { + if self.refuse_at == Some(step) { + return Err(io::Error::other("injected refusal")); + } + self.calls.push(step); + Ok(()) + } +} + +impl RetentionRecoveryStorage for Recording { + fn discard_head_stage(&mut self) -> io::Result<()> { + self.record(Step::DiscardHeadStage) + } + fn discard_manifest_stage(&mut self) -> io::Result<()> { + self.record(Step::DiscardManifestStage) + } + fn discard_root_stage(&mut self) -> io::Result<()> { + self.record(Step::DiscardRootStage) + } + fn link_root(&mut self) -> io::Result<()> { + self.record(Step::LinkRoot) + } + fn link_manifest(&mut self) -> io::Result<()> { + self.record(Step::LinkManifest) + } + fn finalize_head(&mut self) -> io::Result<()> { + self.record(Step::FinalizeHead) + } + fn remove_root_stage(&mut self) -> io::Result<()> { + self.record(Step::RemoveRootStage) + } + fn remove_manifest_stage(&mut self) -> io::Result<()> { + self.record(Step::RemoveManifestStage) + } +} + +const FINALIZE: [Step; 3] = [ + Step::FinalizeHead, + Step::RemoveRootStage, + Step::RemoveManifestStage, +]; + +#[test] +fn every_step_calls_exactly_its_capability_in_plan_order() -> Result<(), Box> { + let all = [ + Step::DiscardHeadStage, + Step::DiscardManifestStage, + Step::DiscardRootStage, + Step::LinkRoot, + Step::LinkManifest, + Step::FinalizeHead, + Step::RemoveRootStage, + Step::RemoveManifestStage, + ]; + let plan = RetentionRecoveryPlan::new(all.to_vec(), RetentionRecoveryOutcome::Committed); + let mut storage = Recording::default(); + + let receipt = execute_retention_recovery(&mut storage, &plan)?; + + assert_eq!(storage.calls, all); + assert_eq!(receipt.executed(), all); + assert_eq!(receipt.outcome(), RetentionRecoveryOutcome::Committed); + Ok(()) +} + +#[test] +fn an_empty_plan_touches_nothing_and_reports_its_outcome() -> Result<(), Box> { + let plan = RetentionRecoveryPlan::new(Vec::new(), RetentionRecoveryOutcome::Clean); + let mut storage = Recording::default(); + + let receipt = execute_retention_recovery(&mut storage, &plan)?; + + assert!(storage.calls.is_empty()); + assert!(receipt.executed().is_empty()); + assert_eq!(receipt.outcome(), RetentionRecoveryOutcome::Clean); + Ok(()) +} + +#[test] +fn a_refused_step_stops_execution_and_names_the_completed_prefix() -> Result<(), Box> { + let plan = RetentionRecoveryPlan::new(FINALIZE.to_vec(), RetentionRecoveryOutcome::Committed); + let mut storage = Recording { + calls: Vec::new(), + refuse_at: Some(Step::RemoveRootStage), + }; + + let error: RetentionRecoveryError = execute_retention_recovery(&mut storage, &plan) + .err() + .ok_or("an injected refusal was reported as success")?; + + assert_eq!(error.step(), Step::RemoveRootStage); + assert_eq!(error.executed(), [Step::FinalizeHead]); + assert_eq!(storage.calls, [Step::FinalizeHead]); + assert!(error.source().is_some()); + Ok(()) +} diff --git a/src/adapters/retention/recovery_storage.rs b/src/adapters/retention/recovery_storage.rs new file mode 100644 index 0000000..5404642 --- /dev/null +++ b/src/adapters/retention/recovery_storage.rs @@ -0,0 +1,72 @@ +//! This module owns the blocking storage capability port for retention recovery. + +use std::io; + +/// Durable capabilities retention recovery executes, one per plan step. +/// +/// Each capability owns its complete effect and the synchronization that makes +/// it durable, so an implementation cannot report a step as done before its +/// evidence would survive process death. Every capability is called at most +/// once per plan, in plan order, and never after a refused capability. +pub trait RetentionRecoveryStorage { + /// Removes a truncated `head.next` and synchronizes `retention`. + /// + /// # Errors + /// + /// Returns the exact filesystem failure; the stage must remain when it fails. + fn discard_head_stage(&mut self) -> io::Result<()>; + + /// Removes a truncated, never linked `manifest.next` and synchronizes `retention`. + /// + /// # Errors + /// + /// Returns the exact filesystem failure; the stage must remain when it fails. + fn discard_manifest_stage(&mut self) -> io::Result<()>; + + /// Removes a truncated, never linked `root.next` and synchronizes `retention`. + /// + /// # Errors + /// + /// Returns the exact filesystem failure; the stage must remain when it fails. + fn discard_root_stage(&mut self) -> io::Result<()>; + + /// Admits the staged root's namespace directory, links the complete root + /// stage into it without replacement, and synchronizes both directories. + /// + /// # Errors + /// + /// Returns the exact filesystem failure or a refusal of a conflicting entry. + fn link_root(&mut self) -> io::Result<()>; + + /// Links the complete manifest stage into the manifest pool without + /// replacement and synchronizes the pool. + /// + /// # Errors + /// + /// Returns the exact filesystem failure or a refusal of a conflicting entry. + fn link_manifest(&mut self) -> io::Result<()>; + + /// Replaces `retention/HEAD` with the complete head stage atomically and + /// synchronizes `retention`. + /// + /// # Errors + /// + /// Returns the exact filesystem failure. + fn finalize_head(&mut self) -> io::Result<()>; + + /// Removes the retained root stage after proving its pool link and + /// synchronizes `retention`. + /// + /// # Errors + /// + /// Returns the exact filesystem failure or a refusal when the link is not proven. + fn remove_root_stage(&mut self) -> io::Result<()>; + + /// Removes the retained manifest stage after proving its pool link and + /// synchronizes `retention`. + /// + /// # Errors + /// + /// Returns the exact filesystem failure or a refusal when the link is not proven. + fn remove_manifest_stage(&mut self) -> io::Result<()>; +} diff --git a/src/lib.rs b/src/lib.rs index f07545d..68e493d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -143,15 +143,16 @@ pub use adapters::{ RetentionPool, RetentionPoolEntryObservation, RetentionPoolObservations, RetentionPublicationError, RetentionPublicationOutcome, RetentionPublicationPhase, RetentionPublicationPreparation, RetentionPublicationPreparationError, - RetentionPublicationReceipt, RetentionPublicationStorage, RetentionRecoveryEvidence, - RetentionRecoveryOutcome, RetentionRecoveryPlan, RetentionRecoveryRefusal, - RetentionRecoveryStep, RetentionRootDecodeError, RetentionRootEncodeError, + RetentionPublicationReceipt, RetentionPublicationStorage, RetentionRecoveryError, + RetentionRecoveryEvidence, RetentionRecoveryOutcome, RetentionRecoveryPlan, + RetentionRecoveryReceipt, RetentionRecoveryRefusal, RetentionRecoveryStep, + RetentionRecoveryStorage, RetentionRootDecodeError, RetentionRootEncodeError, RetentionRootStageAssessment, RetentionStageAssessment, RetentionStageAssessments, RetentionTransitionDisposition, RetentionTransitionError, RetentionTransitionPreflight, RetentionTransitionPreflightError, RetentionTransitionReadiness, VerifiedRetentionClosure, assess_head_stage, assess_manifest_stage, assess_root_stage, execute_retention_publication, - plan_retention_recovery, plan_retention_transition, preflight_retention_transition, - prepare_retention_publication, verify_retention_closure, + execute_retention_recovery, plan_retention_recovery, plan_retention_transition, + preflight_retention_transition, prepare_retention_publication, verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, From 16f22a92191a9fe4fe008d2c669fbfb08760679b Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 9 Sep 2026 12:34:24 -0700 Subject: [PATCH 3/8] Add: recover retained retention stages under filesystem authority FilesystemRetentionPublicationAuthority::recover observes the published state, reads root.next, manifest.next, and head.next within their format bounds (one byte past the bound so an oversized stage is corrupt rather than truncated), looks up the pool entries the complete stages name, plans through plan_retention_recovery, and executes the plan as the RetentionRecoveryStorage implementation under the retained writer lock. Complete stages are reopened through the new FilesystemRetentionStage::reopen, which binds the handle and the named entry to their identity exactly as a freshly created stage is, so link, replace, and remove refuse a substituted stage during recovery too. A truncated stage is discarded only after its kind, length, and identity match what was observed. Every step synchronizes the directory it changed before returning. Four laws build real crash prefixes by driving the publication phases directly and stopping: a clean store is clean; a root stage written and synchronized is linked and retained as a protected orphan; a head stage synchronized before the crash is finalized, the stages are removed, and the byte-identical retry reports AlreadyCommitted; a truncated root stage is discarded. Publication does not yet call recover itself; that wiring follows. Refs #19 --- CHANGELOG.md | 6 +- src/adapters/filesystem_exact_record.rs | 3 +- src/adapters/retention.rs | 6 + .../filesystem_retention_authority.rs | 3 + .../filesystem_retention_recovery.rs | 267 ++++++++++++++++++ .../filesystem_retention_recovery_error.rs | 48 ++++ ...lesystem_retention_recovery_observation.rs | 154 ++++++++++ .../filesystem_retention_recovery_tests.rs | 146 ++++++++++ .../retention/filesystem_retention_stage.rs | 17 ++ src/lib.rs | 35 +-- 10 files changed, 666 insertions(+), 19 deletions(-) create mode 100644 src/adapters/retention/filesystem_retention_recovery.rs create mode 100644 src/adapters/retention/filesystem_retention_recovery_error.rs create mode 100644 src/adapters/retention/filesystem_retention_recovery_observation.rs create mode 100644 src/adapters/retention/filesystem_retention_recovery_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f8865c..6d94983 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,11 @@ after its public API and format compatibility policies are established. `RetentionRecoveryRefusal`. `RetentionRecoveryStorage` names one blocking capability per step and `execute_retention_recovery` runs a plan in order, stopping at the first refused step with the completed prefix named in - `RetentionRecoveryError`; the filesystem implementation is the next step. + `RetentionRecoveryError`. `FilesystemRetentionPublicationAuthority::recover` + observes the stages within their format bounds, reopens complete stages + bound to their identity, and executes the plan under the retained writer + lock, so a crash after the head stage is synchronized finalizes on restart + and a byte-identical retry is already committed. - `FilesystemRetentionPublicationAuthority` executes the 17 ordered retention publication phases against a completely migrated version-2 root. It stages `root.next`, `manifest.next`, and `head.next` exclusively, verifies device diff --git a/src/adapters/filesystem_exact_record.rs b/src/adapters/filesystem_exact_record.rs index 14beda7..92590bd 100644 --- a/src/adapters/filesystem_exact_record.rs +++ b/src/adapters/filesystem_exact_record.rs @@ -205,7 +205,8 @@ pub(super) fn link_without_replacement( } } -fn open_read(directory: &Dir, name: &str) -> io::Result { +/// Opens `name` read-only without following links or blocking. +pub(super) fn open_read(directory: &Dir, name: &str) -> io::Result { let mut options = OpenOptions::new(); options.read(true).follow(FollowSymlinks::No).nonblock(true); directory.open_with(name, &options) diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 5a75baf..5c1061f 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -36,6 +36,11 @@ mod filesystem_retention_namespace; #[cfg(test)] mod filesystem_retention_namespace_tests; mod filesystem_retention_pool_name; +mod filesystem_retention_recovery; +mod filesystem_retention_recovery_error; +mod filesystem_retention_recovery_observation; +#[cfg(test)] +mod filesystem_retention_recovery_tests; mod filesystem_retention_refusal; mod filesystem_retention_stage; mod filesystem_retention_storage; @@ -115,6 +120,7 @@ pub use filesystem_retention_authority_error::{ FilesystemRetentionAuthorityError, RetentionAuthorityDirectory, }; pub use filesystem_retention_current::ObservedRetentionState; +pub use filesystem_retention_recovery_error::FilesystemRetentionRecoveryError; pub use filesystem_retention_refusal::RetentionCurrentStateRefusal; pub use head_decode_error::RetentionHeadDecodeError; pub use manifest_decode_error::RetentionManifestDecodeError; diff --git a/src/adapters/retention/filesystem_retention_authority.rs b/src/adapters/retention/filesystem_retention_authority.rs index ada4ed5..fe0cf1e 100644 --- a/src/adapters/retention/filesystem_retention_authority.rs +++ b/src/adapters/retention/filesystem_retention_authority.rs @@ -9,6 +9,7 @@ use super::filesystem_retention_authority_error::{ FilesystemRetentionAuthorityError as Error, RetentionAuthorityDirectory as Directory, }; use super::filesystem_retention_current::{self, ObservedRetentionState}; +use super::filesystem_retention_recovery::RetentionRecoveryContext; use crate::adapters::{FilesystemVersionTwoAdmission, FilesystemWriterLock}; /// Exclusive authority to publish retention transitions on one pinned root. @@ -32,6 +33,7 @@ pub struct FilesystemRetentionPublicationAuthority { pub(super) roots: Dir, pub(super) manifests: Dir, pub(super) attempt: Option, + pub(super) recovery: Option, _lock: FilesystemWriterLock, } @@ -68,6 +70,7 @@ impl FilesystemRetentionPublicationAuthority { roots, manifests, attempt: None, + recovery: None, _lock: lock, }) } diff --git a/src/adapters/retention/filesystem_retention_recovery.rs b/src/adapters/retention/filesystem_retention_recovery.rs new file mode 100644 index 0000000..1de0e93 --- /dev/null +++ b/src/adapters/retention/filesystem_retention_recovery.rs @@ -0,0 +1,267 @@ +//! This module owns filesystem execution of retention recovery under authority. + +use std::io; + +use cap_fs_ext::DirExt; +use cap_std::fs::Dir; + +use super::filesystem_retention_authority::FilesystemRetentionPublicationAuthority; +use super::filesystem_retention_pool_name as pool_name; +use super::filesystem_retention_recovery_observation::{RetentionRecoveryObservation, StageBytes}; +use super::filesystem_retention_stage::{FilesystemRetentionStage, invalid_data}; +use super::{ + FilesystemRetentionRecoveryError as Error, RetentionRecoveryReceipt, RetentionRecoveryStorage, + RetentionStageAssessment, assess_head_stage, assess_manifest_stage, assess_root_stage, + execute_retention_recovery, plan_retention_recovery, +}; +use crate::adapters::filesystem_catalog_artifact::synchronize_directory; +use crate::adapters::filesystem_exact_record::{self as exact_record, EntryIdentity}; + +/// One retained stage as recovery holds it between steps. +pub(super) enum RecoveredStage { + /// A complete stage reopened and bound to its identity. + Complete { + stage: FilesystemRetentionStage, + pool_name: String, + namespace: Option, + }, + /// A truncated stage identified for discard. + Truncated { + identity: EntryIdentity, + length: u64, + }, +} + +/// The retained stages one recovery run operates on. +pub(super) struct RetentionRecoveryContext { + root: Option, + manifest: Option, + head: Option, +} + +impl RetentionRecoveryContext { + fn reopen(retention: &Dir, observation: &RetentionRecoveryObservation) -> io::Result { + let root = observation + .root() + .map(|stage| -> io::Result { + match assess_root_stage(Some(&stage.bytes)) { + RetentionStageAssessment::Complete(admitted) => Ok(RecoveredStage::Complete { + stage: FilesystemRetentionStage::reopen( + retention, + pool_name::ROOT_STAGE, + &stage.bytes, + )?, + pool_name: pool_name::root(admitted.root().generation(), admitted.digest()), + namespace: Some(pool_name::namespace(admitted.root().namespace().digest())), + }), + _ => Ok(truncated(stage)), + } + }) + .transpose()?; + let manifest = observation + .manifest() + .map(|stage| -> io::Result { + match assess_manifest_stage(Some(&stage.bytes)) { + RetentionStageAssessment::Complete(admitted) => Ok(RecoveredStage::Complete { + stage: FilesystemRetentionStage::reopen( + retention, + pool_name::MANIFEST_STAGE, + &stage.bytes, + )?, + pool_name: pool_name::manifest( + admitted.manifest().generation(), + admitted.digest(), + ), + namespace: None, + }), + _ => Ok(truncated(stage)), + } + }) + .transpose()?; + let head = observation + .head() + .map(|stage| -> io::Result { + match assess_head_stage(Some(&stage.bytes)) { + RetentionStageAssessment::Complete(_) => Ok(RecoveredStage::Complete { + stage: FilesystemRetentionStage::reopen( + retention, + pool_name::HEAD_STAGE, + &stage.bytes, + )?, + pool_name: pool_name::HEAD.to_owned(), + namespace: None, + }), + _ => Ok(truncated(stage)), + } + }) + .transpose()?; + Ok(Self { + root, + manifest, + head, + }) + } +} + +fn truncated(stage: &StageBytes) -> RecoveredStage { + RecoveredStage::Truncated { + identity: stage.identity, + length: u64::try_from(stage.bytes.len()).unwrap_or(u64::MAX), + } +} + +impl FilesystemRetentionPublicationAuthority { + /// Observes, plans, and executes recovery of every fixed retention stage. + /// + /// The synchronous call runs under the retained writer lock. A clean store + /// returns an empty receipt; a truncated pre-effect stage is discarded; a + /// complete stage is linked and retained as a recovery-protected orphan; + /// a complete head over linked stages is finalized. Any pending + /// publication attempt is discarded first, and the caller re-verifies + /// current state afterwards. + /// + /// # Errors + /// + /// Returns [`FilesystemRetentionRecoveryError`](super::FilesystemRetentionRecoveryError) + /// at the exact observation failure, planning refusal, or refused step. + pub fn recover(&mut self) -> Result { + self.attempt = None; + self.recovery = None; + let observation = + RetentionRecoveryObservation::observe(&self.retention, &self.roots, &self.manifests) + .map_err(|source| Error::Observe { source })?; + let plan = plan_retention_recovery(observation.evidence()) + .map_err(|source| Error::Plan { source })?; + self.recovery = Some( + RetentionRecoveryContext::reopen(&self.retention, &observation) + .map_err(|source| Error::Observe { source })?, + ); + let result = + execute_retention_recovery(self, &plan).map_err(|source| Error::Execute { source }); + self.recovery = None; + result + } +} + +fn no_recovery() -> io::Error { + invalid_data("no retention recovery is in progress") +} + +fn take_complete( + slot: &mut Option, +) -> io::Result<(FilesystemRetentionStage, String, Option)> { + match slot.take() { + Some(RecoveredStage::Complete { + stage, + pool_name, + namespace, + }) => Ok((stage, pool_name, namespace)), + Some(other) => { + *slot = Some(other); + Err(invalid_data("recovery step expected a complete stage")) + } + None => Err(invalid_data("recovery step expected a retained stage")), + } +} + +fn discard_truncated( + retention: &Dir, + name: &str, + slot: &mut Option, +) -> io::Result<()> { + let Some(RecoveredStage::Truncated { identity, length }) = slot.take() else { + return Err(invalid_data("recovery step expected a truncated stage")); + }; + let metadata = retention.symlink_metadata(name)?; + if !metadata.is_file() || metadata.len() != length || EntryIdentity::from(&metadata) != identity + { + return Err(invalid_data( + "truncated retention stage changed before discard", + )); + } + retention.remove_file(name)?; + exact_record::require_absent(retention, name) + .map_err(|_source| invalid_data("discarded retention stage remained visible"))?; + synchronize_directory(retention) +} + +impl RetentionRecoveryStorage for FilesystemRetentionPublicationAuthority { + fn discard_head_stage(&mut self) -> io::Result<()> { + let context = self.recovery.as_mut().ok_or_else(no_recovery)?; + discard_truncated(&self.retention, pool_name::HEAD_STAGE, &mut context.head) + } + + fn discard_manifest_stage(&mut self) -> io::Result<()> { + let context = self.recovery.as_mut().ok_or_else(no_recovery)?; + discard_truncated( + &self.retention, + pool_name::MANIFEST_STAGE, + &mut context.manifest, + ) + } + + fn discard_root_stage(&mut self) -> io::Result<()> { + let context = self.recovery.as_mut().ok_or_else(no_recovery)?; + discard_truncated(&self.retention, pool_name::ROOT_STAGE, &mut context.root) + } + + fn link_root(&mut self) -> io::Result<()> { + let context = self.recovery.as_ref().ok_or_else(no_recovery)?; + let Some(RecoveredStage::Complete { + stage, + pool_name: name, + namespace: Some(namespace), + }) = context.root.as_ref() + else { + return Err(invalid_data("link_root expected a complete root stage")); + }; + match self.roots.create_dir(namespace) { + Ok(()) => {} + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => {} + Err(source) => return Err(source), + } + let directory = self.roots.open_dir_nofollow(namespace)?; + synchronize_directory(&self.roots)?; + stage.link(&self.retention, &directory, name)?; + synchronize_directory(&directory) + } + + fn link_manifest(&mut self) -> io::Result<()> { + let context = self.recovery.as_ref().ok_or_else(no_recovery)?; + let Some(RecoveredStage::Complete { + stage, + pool_name: name, + .. + }) = context.manifest.as_ref() + else { + return Err(invalid_data( + "link_manifest expected a complete manifest stage", + )); + }; + stage.link(&self.retention, &self.manifests, name)?; + synchronize_directory(&self.manifests) + } + + fn finalize_head(&mut self) -> io::Result<()> { + let context = self.recovery.as_mut().ok_or_else(no_recovery)?; + let (stage, _name, _namespace) = take_complete(&mut context.head)?; + stage.replace(&self.retention, pool_name::HEAD)?; + synchronize_directory(&self.retention) + } + + fn remove_root_stage(&mut self) -> io::Result<()> { + let context = self.recovery.as_mut().ok_or_else(no_recovery)?; + let (stage, name, namespace) = take_complete(&mut context.root)?; + let namespace = namespace.ok_or_else(|| invalid_data("root stage without a namespace"))?; + let directory = self.roots.open_dir_nofollow(&namespace)?; + stage.remove(&self.retention, &directory, &name)?; + synchronize_directory(&self.retention) + } + + fn remove_manifest_stage(&mut self) -> io::Result<()> { + let context = self.recovery.as_mut().ok_or_else(no_recovery)?; + let (stage, name, _namespace) = take_complete(&mut context.manifest)?; + stage.remove(&self.retention, &self.manifests, &name)?; + synchronize_directory(&self.retention) + } +} diff --git a/src/adapters/retention/filesystem_retention_recovery_error.rs b/src/adapters/retention/filesystem_retention_recovery_error.rs new file mode 100644 index 0000000..64c4513 --- /dev/null +++ b/src/adapters/retention/filesystem_retention_recovery_error.rs @@ -0,0 +1,48 @@ +//! This module owns the typed error of filesystem retention recovery. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{RetentionRecoveryError, RetentionRecoveryRefusal}; + +/// Why filesystem retention recovery did not reach a receipt. +#[derive(Debug)] +#[non_exhaustive] +pub enum FilesystemRetentionRecoveryError { + /// Reading the current state, a stage, or a pool entry failed. + Observe { + /// The exact filesystem or admission failure. + source: io::Error, + }, + /// The observed stages are unrecoverable ambiguity. + Plan { + /// The exact planning refusal. + source: RetentionRecoveryRefusal, + }, + /// A recovery step refused; earlier steps' effects remain. + Execute { + /// The refused step, the completed prefix, and the storage error. + source: RetentionRecoveryError, + }, +} + +impl fmt::Display for FilesystemRetentionRecoveryError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Observe { .. } => "retention recovery could not observe the store", + Self::Plan { .. } => "retention recovery refused the observed stages", + Self::Execute { .. } => "a retention recovery step refused", + }) + } +} + +impl Error for FilesystemRetentionRecoveryError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Observe { source } => Some(source), + Self::Plan { source } => Some(source), + Self::Execute { source } => Some(source), + } + } +} diff --git a/src/adapters/retention/filesystem_retention_recovery_observation.rs b/src/adapters/retention/filesystem_retention_recovery_observation.rs new file mode 100644 index 0000000..9b21fc8 --- /dev/null +++ b/src/adapters/retention/filesystem_retention_recovery_observation.rs @@ -0,0 +1,154 @@ +//! This module owns restart observation of the retention stages and pools. + +use std::io::{self, Read}; + +use cap_fs_ext::DirExt; +use cap_std::fs::Dir; + +use super::filesystem_retention_current::{self, ObservedRetentionState}; +use super::filesystem_retention_pool_name as pool_name; +use super::{ + RetentionPoolEntryObservation as Pool, RetentionPoolObservations, RetentionRecoveryEvidence, + RetentionStageAssessment, RetentionStageAssessments, assess_head_stage, assess_manifest_stage, + assess_root_stage, head_decoder, root_header_decoder, +}; +use crate::adapters::filesystem_exact_record::{ + self as exact_record, EntryIdentity, ExactRecordError, +}; + +/// 160-byte header, 4,096 entries of 72 bytes, manifest digest, checksum. +const MANIFEST_MAXIMUM_ENCODED_LENGTH: usize = 295_136; + +/// The exact bytes and entry identity of one retained stage. +pub(super) struct StageBytes { + pub(super) bytes: Box<[u8]>, + pub(super) identity: EntryIdentity, +} + +/// Everything restart read under writer authority before planning recovery. +pub(super) struct RetentionRecoveryObservation { + current: Option, + root: Option, + manifest: Option, + head: Option, + pools: RetentionPoolObservations, +} + +impl RetentionRecoveryObservation { + /// Reads the current state, the three stages, and the pool entries the + /// complete stages name. Performs no mutation. + pub(super) fn observe(retention: &Dir, roots: &Dir, manifests: &Dir) -> io::Result { + let current = filesystem_retention_current::observe(retention, manifests)?; + let root = read_stage( + retention, + pool_name::ROOT_STAGE, + root_header_decoder::MAXIMUM_ENCODED_LENGTH, + )?; + let manifest = read_stage( + retention, + pool_name::MANIFEST_STAGE, + MANIFEST_MAXIMUM_ENCODED_LENGTH, + )?; + let head = read_stage( + retention, + pool_name::HEAD_STAGE, + head_decoder::ENCODED_LENGTH, + )?; + let root_pool = match assess_root_stage(root.as_ref().map(|stage| &*stage.bytes)) { + RetentionStageAssessment::Complete(admitted) => { + let namespace = pool_name::namespace(admitted.root().namespace().digest()); + let name = pool_name::root(admitted.root().generation(), admitted.digest()); + match roots.open_dir_nofollow(namespace) { + Ok(directory) => pool_entry(&directory, &name, admitted.encoded())?, + Err(source) if source.kind() == io::ErrorKind::NotFound => Pool::Absent, + Err(source) => return Err(source), + } + } + _ => Pool::Absent, + }; + let manifest_pool = + match assess_manifest_stage(manifest.as_ref().map(|stage| &*stage.bytes)) { + RetentionStageAssessment::Complete(admitted) => { + let name = + pool_name::manifest(admitted.manifest().generation(), admitted.digest()); + pool_entry(manifests, &name, admitted.encoded())? + } + _ => Pool::Absent, + }; + Ok(Self { + current, + root, + manifest, + head, + pools: RetentionPoolObservations { + root: root_pool, + manifest: manifest_pool, + }, + }) + } + + /// The pure evidence recovery plans from. + pub(super) fn evidence(&self) -> RetentionRecoveryEvidence<'_, '_> { + RetentionRecoveryEvidence::new( + self.current.as_ref(), + RetentionStageAssessments { + root: assess_root_stage(self.root.as_ref().map(|stage| &*stage.bytes)), + manifest: assess_manifest_stage(self.manifest.as_ref().map(|stage| &*stage.bytes)), + head: assess_head_stage(self.head.as_ref().map(|stage| &*stage.bytes)), + }, + self.pools, + ) + } + + pub(super) const fn root(&self) -> Option<&StageBytes> { + self.root.as_ref() + } + + pub(super) const fn manifest(&self) -> Option<&StageBytes> { + self.manifest.as_ref() + } + + pub(super) const fn head(&self) -> Option<&StageBytes> { + self.head.as_ref() + } +} + +/// Reads a stage's complete bytes up to one byte past `bound`. +/// +/// A stage longer than its format's maximum is returned in full up to that +/// point so assessment classifies it as corrupt rather than truncated. +fn read_stage(retention: &Dir, name: &str, bound: usize) -> io::Result> { + let mut file = match exact_record::open_read(retention, name) { + Ok(file) => file, + Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(source) => return Err(source), + }; + let metadata = file.metadata()?; + if !metadata.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "retained retention stage is not a regular file", + )); + } + let identity = EntryIdentity::from(&metadata); + let limit = bound + .checked_add(1) + .and_then(|limit| u64::try_from(limit).ok()) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "stage bound overflowed"))?; + let mut bytes = Vec::new(); + file.by_ref().take(limit).read_to_end(&mut bytes)?; + Ok(Some(StageBytes { + bytes: bytes.into_boxed_slice(), + identity, + })) +} + +/// Whether `directory` holds `name` with exactly `expected` bytes. +fn pool_entry(directory: &Dir, name: &str, expected: &[u8]) -> io::Result { + match exact_record::read_exact_optional(directory, name, expected.len()) { + Ok(None) => Ok(Pool::Absent), + Ok(Some(bytes)) if bytes == expected => Ok(Pool::Identical), + Ok(Some(_)) | Err(ExactRecordError::Refused(_)) => Ok(Pool::Different), + Err(ExactRecordError::Io(source)) => Err(source), + } +} diff --git a/src/adapters/retention/filesystem_retention_recovery_tests.rs b/src/adapters/retention/filesystem_retention_recovery_tests.rs new file mode 100644 index 0000000..0cb5d3a --- /dev/null +++ b/src/adapters/retention/filesystem_retention_recovery_tests.rs @@ -0,0 +1,146 @@ +//! Filesystem retention recovery laws over real crash prefixes. + +use std::error::Error; +use std::fs; + +use super::filesystem_retention_test_fixture::{ + ROOT_HEX, fixture, head_path, initial_preparation, manifest_pool_path, open_authority, + root_pool_path, +}; +use super::{ + FilesystemRetentionPublicationAuthority, RetentionPublicationOutcome, + RetentionPublicationPreparation, RetentionPublicationStorage, + RetentionRecoveryOutcome as Outcome, RetentionRecoveryStep as Step, +}; +use crate::execute_retention_publication; + +type Phase<'a> = + &'a mut dyn FnMut(&mut FilesystemRetentionPublicationAuthority) -> std::io::Result<()>; + +/// Executes publication phases 1 through `count` and stops, like a crash there. +fn drive( + authority: &mut FilesystemRetentionPublicationAuthority, + preparation: &RetentionPublicationPreparation<'_>, + count: usize, +) -> Result<(), Box> { + let publication = preparation + .publication() + .ok_or("preparation carries no publication")?; + let root = preparation.candidate(); + let phases: [Phase<'_>; 13] = [ + &mut |a| a.verify_current(preparation).map(|_| ()), + &mut |a| a.write_root_stage(root), + &mut |a| a.synchronize_root_stage(), + &mut |a| a.admit_root_namespace(root).map(|_| ()), + &mut |a| a.synchronize_roots_after_namespace(), + &mut |a| a.link_root(root), + &mut |a| a.synchronize_root_namespace(root), + &mut |a| a.write_manifest_stage(publication.manifest()), + &mut |a| a.synchronize_manifest_stage(), + &mut |a| a.link_manifest(publication.manifest()), + &mut |a| a.synchronize_manifest_pool(), + &mut |a| a.write_head_stage(publication.head()), + &mut |a| a.synchronize_head_stage(), + ]; + for phase in phases.into_iter().take(count) { + phase(authority)?; + } + Ok(()) +} + +#[test] +fn a_clean_store_recovers_to_clean() -> Result<(), Box> { + let (_sandbox, mut authority) = open_authority("filesystem-retention-recovery-clean")?; + let receipt = authority.recover()?; + assert!(receipt.executed().is_empty()); + assert_eq!(receipt.outcome(), Outcome::Clean); + Ok(()) +} + +#[test] +fn a_written_root_stage_is_linked_and_protected() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-recovery-root")?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + drive(&mut authority, &preparation, 3)?; + + let receipt = authority.recover()?; + + assert_eq!(receipt.executed(), [Step::LinkRoot]); + assert_eq!( + receipt.outcome(), + Outcome::Protected { + root_stage: true, + manifest_stage: false + } + ); + assert_eq!( + fs::read(root_pool_path(sandbox.path(), preparation.candidate()))?, + root_bytes + ); + assert!(sandbox.path().join("retention").join("root.next").is_file()); + assert!(!head_path(sandbox.path()).exists()); + Ok(()) +} + +#[test] +fn a_synchronized_head_stage_is_finalized_and_the_retry_is_already_committed() +-> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-recovery-head")?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + drive(&mut authority, &preparation, 13)?; + let publication = preparation.publication().ok_or("no publication")?; + + let receipt = authority.recover()?; + + assert_eq!( + receipt.executed(), + [ + Step::FinalizeHead, + Step::RemoveRootStage, + Step::RemoveManifestStage + ] + ); + assert_eq!(receipt.outcome(), Outcome::Committed); + assert_eq!( + fs::read(head_path(sandbox.path()))?, + publication.head().encoded() + ); + assert_eq!( + fs::read(manifest_pool_path(sandbox.path(), &preparation))?, + publication.manifest().encoded() + ); + for stage in ["root.next", "manifest.next", "head.next"] { + assert!( + !sandbox.path().join("retention").join(stage).exists(), + "{stage} remained" + ); + } + let retry = execute_retention_publication(&mut authority, &preparation)?; + assert_eq!( + retry.outcome(), + RetentionPublicationOutcome::AlreadyCommitted + ); + Ok(()) +} + +#[test] +fn a_truncated_root_stage_is_discarded() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-recovery-truncated")?; + let root_bytes = fixture(ROOT_HEX)?; + let stage = sandbox.path().join("retention").join("root.next"); + fs::write( + &stage, + root_bytes + .get(..100) + .ok_or("root fixture shorter than 100 bytes")?, + )?; + + let receipt = authority.recover()?; + + assert_eq!(receipt.executed(), [Step::DiscardRootStage]); + assert_eq!(receipt.outcome(), Outcome::Clean); + assert!(!stage.exists()); + Ok(()) +} diff --git a/src/adapters/retention/filesystem_retention_stage.rs b/src/adapters/retention/filesystem_retention_stage.rs index 35a2459..0167b8c 100644 --- a/src/adapters/retention/filesystem_retention_stage.rs +++ b/src/adapters/retention/filesystem_retention_stage.rs @@ -37,6 +37,23 @@ impl FilesystemRetentionStage { }) } + /// Reopens a retained stage whose exact bytes restart already read. + /// + /// The handle and the named entry are verified against `expected` and + /// bound to the entry's identity, so every later transition refuses a + /// substituted or replaced stage exactly as a freshly created one would. + pub(super) fn reopen(root: &Dir, name: &'static str, expected: &[u8]) -> io::Result { + let file = exact_record::open_read(root, name)?; + let identity = EntryIdentity::of_file(&file)?; + verify_named_record(root, name, expected, identity)?; + Ok(Self { + name, + expected: Box::from(expected), + identity, + file, + }) + } + /// Synchronizes the complete stage and reverifies its exact bytes. pub(super) fn synchronize(&self, root: &Dir) -> io::Result<()> { self.require_handle()?; diff --git a/src/lib.rs b/src/lib.rs index 68e493d..c6ff84e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -136,23 +136,24 @@ pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, FilesystemRetentionAuthorityError, FilesystemRetentionPublicationAuthority, - ObservedRetentionState, PreparedRetentionPublication, RetentionAuthorityDirectory, - RetentionClosureVerificationError, RetentionCurrentStateRefusal, RetentionFixedStage, - RetentionHeadDecodeError, RetentionHeadStageAssessment, RetentionManifestDecodeError, - RetentionManifestEncodeError, RetentionManifestStageAssessment, RetentionNamespaceAdmission, - RetentionPool, RetentionPoolEntryObservation, RetentionPoolObservations, - RetentionPublicationError, RetentionPublicationOutcome, RetentionPublicationPhase, - RetentionPublicationPreparation, RetentionPublicationPreparationError, - RetentionPublicationReceipt, RetentionPublicationStorage, RetentionRecoveryError, - RetentionRecoveryEvidence, RetentionRecoveryOutcome, RetentionRecoveryPlan, - RetentionRecoveryReceipt, RetentionRecoveryRefusal, RetentionRecoveryStep, - RetentionRecoveryStorage, RetentionRootDecodeError, RetentionRootEncodeError, - RetentionRootStageAssessment, RetentionStageAssessment, RetentionStageAssessments, - RetentionTransitionDisposition, RetentionTransitionError, RetentionTransitionPreflight, - RetentionTransitionPreflightError, RetentionTransitionReadiness, VerifiedRetentionClosure, - assess_head_stage, assess_manifest_stage, assess_root_stage, execute_retention_publication, - execute_retention_recovery, plan_retention_recovery, plan_retention_transition, - preflight_retention_transition, prepare_retention_publication, verify_retention_closure, + FilesystemRetentionRecoveryError, ObservedRetentionState, PreparedRetentionPublication, + RetentionAuthorityDirectory, RetentionClosureVerificationError, RetentionCurrentStateRefusal, + RetentionFixedStage, RetentionHeadDecodeError, RetentionHeadStageAssessment, + RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionManifestStageAssessment, + RetentionNamespaceAdmission, RetentionPool, RetentionPoolEntryObservation, + RetentionPoolObservations, RetentionPublicationError, RetentionPublicationOutcome, + RetentionPublicationPhase, RetentionPublicationPreparation, + RetentionPublicationPreparationError, RetentionPublicationReceipt, RetentionPublicationStorage, + RetentionRecoveryError, RetentionRecoveryEvidence, RetentionRecoveryOutcome, + RetentionRecoveryPlan, RetentionRecoveryReceipt, RetentionRecoveryRefusal, + RetentionRecoveryStep, RetentionRecoveryStorage, RetentionRootDecodeError, + RetentionRootEncodeError, RetentionRootStageAssessment, RetentionStageAssessment, + RetentionStageAssessments, RetentionTransitionDisposition, RetentionTransitionError, + RetentionTransitionPreflight, RetentionTransitionPreflightError, RetentionTransitionReadiness, + VerifiedRetentionClosure, assess_head_stage, assess_manifest_stage, assess_root_stage, + execute_retention_publication, execute_retention_recovery, plan_retention_recovery, + plan_retention_transition, preflight_retention_transition, prepare_retention_publication, + verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, From 7e6cf879b63ab8cfd99ac5e9a3ad2feabbd4fc8a Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 9 Sep 2026 13:01:08 -0700 Subject: [PATCH 4/8] Test: recover every retention publication crash prefix to its documented state The retention fixture now drives all 18 storage-port phases in RetentionPublicationPhase::ALL order and stops after any prefix, which is the exact state a process death after that phase leaves behind. Three laws use it. The first walks every prefix from 0 through 18 in a fresh store and requires the documented recovery steps and outcome, the stages left behind, idempotent re-recovery, and the forward retry's result: published after a clean prefix, refused as recovery-required while protected orphans remain, already committed once the head is finalized. The second truncates each stage mid-write and requires only that stage discarded. The third replays successor prefixes over a published generation and requires the committed head to name the successor. recovery.md states that storage execution now exists and only process-death evidence remains; the RETENTION-007 ledger cell names the laws. Refs #19 --- CHANGELOG.md | 6 +- docs/formats/segment-store-v2/recovery.md | 4 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/retention.rs | 2 + ...esystem_retention_recovery_prefix_tests.rs | 208 ++++++++++++++++++ .../filesystem_retention_recovery_tests.rs | 46 +--- .../filesystem_retention_test_fixture.rs | 47 ++++ 7 files changed, 271 insertions(+), 44 deletions(-) create mode 100644 src/adapters/retention/filesystem_retention_recovery_prefix_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d94983..2e7cb39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,11 @@ after its public API and format compatibility policies are established. observes the stages within their format bounds, reopens complete stages bound to their identity, and executes the plan under the retained writer lock, so a crash after the head stage is synchronized finalizes on restart - and a byte-identical retry is already committed. + and a byte-identical retry is already committed. Laws drive every + publication prefix from 0 through 18 phases, truncate each stage mid-write, + and replay successor prefixes over a published generation; each recovers to + its documented state, recovery is idempotent, and the forward retry reports + the predicted outcome. - `FilesystemRetentionPublicationAuthority` executes the 17 ordered retention publication phases against a completely migrated version-2 root. It stages `root.next`, `manifest.next`, and `head.next` exclusively, verifies device diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index efa539b..d4449ad 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -232,7 +232,9 @@ The retention crash points are: | `KEEP-CRASH-052` | retention cleanup synchronization | `RetentionPublicationPhase::ALL` freezes this exact order as a typed public -vocabulary. Storage execution and process-death evidence remain unimplemented. +vocabulary. `FilesystemRetentionPublicationAuthority::recover` implements the +classification above and its effects; process-death evidence remains +unimplemented. Each point requires before, during, and after process-death evidence. Restart must establish exact catalog visibility, retention head, namespace generation, diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index ff8ae2f..1fbda2d 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -15,7 +15,7 @@ case is not evidence. | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; writer-locked initial filesystem publication in `filesystem_retention_storage_tests`; observed-head successor publication, exact predecessor binding, and absent-head refusal in `filesystem_retention_successor_tests`; the store's catalog head must name the closure's catalog generation and digest before any forward write in `filesystem_retention_catalog_tests`; a head whose predecessor disagrees with its manifest refuses in `filesystem_retention_current_tests`; a successor reopens and decodes the manifest-selected predecessor root and refuses an absent or changed one in `filesystem_retention_expectation_tests` | Implemented | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; production 17-phase forward filesystem execution, exclusive staging, byte-equal inode-substitution refusal, and retained-stage recovery refusal in `filesystem_retention_storage_tests`; orphan namespace directories count against the 4,096 ceiling and refuse a new namespace before any stage is written in `filesystem_retention_capacity_tests`; crash injection remains | In progress in #19 | -| `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | recovery-required refusals before any mutation in `filesystem_retention_expectation_tests`: an absent head over populated pools, a non-initial head prepared against an absent head, an orphan directory for a namespace expected absent, and an absent directory for a namespace expected current; debug and release crash matrix remains; replaced protocol directories, an absent or changed head-selected catalog, an over-full census, zero-generation pool names, and a stage retained by a failed write refuse in `filesystem_retention_*_tests`; storage-independent classification of every fixed-stage crash prefix (discard, link and protect, finalize, clean up, or typed refusal) in `recovery_planner_tests` | In progress in #19 | +| `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | recovery-required refusals before any mutation in `filesystem_retention_expectation_tests`: an absent head over populated pools, a non-initial head prepared against an absent head, an orphan directory for a namespace expected absent, and an absent directory for a namespace expected current; debug and release crash matrix remains; replaced protocol directories, an absent or changed head-selected catalog, an over-full census, zero-generation pool names, and a stage retained by a failed write refuse in `filesystem_retention_*_tests`; storage-independent classification of every fixed-stage crash prefix (discard, link and protect, finalize, clean up, or typed refusal) in `recovery_planner_tests`; every publication prefix 0 through 18, each mid-write truncation, and successor prefixes recover in-process to the documented state, idempotently, with the forward retry reporting the predicted outcome, in `filesystem_retention_recovery_prefix_tests` | In progress in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | | `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests`; committed retry reopens the head-selected manifest entry and root pool bytes, refusing absent, changed, or corrupt evidence in `filesystem_retention_current_tests`; every refusal is a typed `RetentionCurrentStateRefusal` source, with superseded, committed-root-absent, committed-root-changed, and head-absent-with-artifacts pinned by downcast | Implemented | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | model-based and source-architecture tests | Planned in #19 | diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 5c1061f..3dabcda 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -40,6 +40,8 @@ mod filesystem_retention_recovery; mod filesystem_retention_recovery_error; mod filesystem_retention_recovery_observation; #[cfg(test)] +mod filesystem_retention_recovery_prefix_tests; +#[cfg(test)] mod filesystem_retention_recovery_tests; mod filesystem_retention_refusal; mod filesystem_retention_stage; diff --git a/src/adapters/retention/filesystem_retention_recovery_prefix_tests.rs b/src/adapters/retention/filesystem_retention_recovery_prefix_tests.rs new file mode 100644 index 0000000..8b0526f --- /dev/null +++ b/src/adapters/retention/filesystem_retention_recovery_prefix_tests.rs @@ -0,0 +1,208 @@ +//! Every publication crash prefix recovers to exactly one documented state. + +use std::error::Error; +use std::fs; +use std::path::Path; + +use super::filesystem_retention_test_fixture::{ + MANIFEST_HEX, PUBLICATION_PHASE_COUNT, ROOT_HEX, drive_publication, fixture, + initial_preparation, open_authority, refusal, successor_preparation, successor_root, +}; +use super::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, RetentionCurrentStateRefusal, + RetentionPublicationError, RetentionPublicationOutcome, RetentionRecoveryOutcome as Outcome, + RetentionRecoveryStep as Step, +}; +use crate::execute_retention_publication; + +const PROTECTED_ROOT: Outcome = Outcome::Protected { + root_stage: true, + manifest_stage: false, +}; +const PROTECTED_BOTH: Outcome = Outcome::Protected { + root_stage: true, + manifest_stage: true, +}; + +/// The documented recovery of a crash after `count` phases, and the forward +/// retry's result afterwards. +fn expected(count: usize) -> (Vec, Outcome, Retry) { + match count { + 0 | 1 => (vec![], Outcome::Clean, Retry::Published), + 2..=5 => (vec![Step::LinkRoot], PROTECTED_ROOT, Retry::Refused), + 6 | 7 => (vec![], PROTECTED_ROOT, Retry::Refused), + 8 | 9 => (vec![Step::LinkManifest], PROTECTED_BOTH, Retry::Refused), + 10 | 11 => (vec![], PROTECTED_BOTH, Retry::Refused), + 12 | 13 => ( + vec![ + Step::FinalizeHead, + Step::RemoveRootStage, + Step::RemoveManifestStage, + ], + Outcome::Committed, + Retry::AlreadyCommitted, + ), + 14 | 15 => ( + vec![Step::RemoveRootStage, Step::RemoveManifestStage], + Outcome::Committed, + Retry::AlreadyCommitted, + ), + 16 => ( + vec![Step::RemoveManifestStage], + Outcome::Committed, + Retry::AlreadyCommitted, + ), + _ => (vec![], Outcome::Clean, Retry::AlreadyCommitted), + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Retry { + Published, + AlreadyCommitted, + Refused, +} + +fn stages_present(root: &Path) -> [bool; 3] { + let retention = root.join("retention"); + ["root.next", "manifest.next", "head.next"].map(|stage| retention.join(stage).exists()) +} + +fn stages_for(outcome: Outcome) -> [bool; 3] { + match outcome { + Outcome::Clean | Outcome::Committed => [false, false, false], + Outcome::Protected { + root_stage, + manifest_stage, + } => [root_stage, manifest_stage, false], + } +} + +#[test] +fn every_initial_publication_prefix_recovers_to_its_documented_state() -> Result<(), Box> +{ + for count in 0..=PUBLICATION_PHASE_COUNT { + let name = format!("filesystem-retention-recovery-prefix-{count}"); + let (sandbox, mut authority) = open_authority(&name)?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + drive_publication(&mut authority, &preparation, count)?; + let (steps, outcome, retry) = expected(count); + + let receipt = authority + .recover() + .map_err(|error| format!("prefix {count}: {error}"))?; + + assert_eq!(receipt.executed(), steps, "prefix {count}: steps"); + assert_eq!(receipt.outcome(), outcome, "prefix {count}: outcome"); + assert_eq!( + stages_present(sandbox.path()), + stages_for(outcome), + "prefix {count}: stages" + ); + let second = authority + .recover() + .map_err(|error| format!("prefix {count} again: {error}"))?; + assert!( + second.executed().is_empty(), + "prefix {count}: recovery is idempotent" + ); + match ( + retry, + execute_retention_publication(&mut authority, &preparation), + ) { + (Retry::Published, Ok(receipt)) => { + assert_eq!(receipt.outcome(), RetentionPublicationOutcome::Published); + } + (Retry::AlreadyCommitted, Ok(receipt)) => { + assert_eq!( + receipt.outcome(), + RetentionPublicationOutcome::AlreadyCommitted + ); + } + (Retry::Refused, Err(RetentionPublicationError::CurrentVerification { source })) => { + assert!( + matches!( + refusal(&source), + Some(RetentionCurrentStateRefusal::RetainedStage) + ), + "prefix {count}: protected orphans refuse forward publication" + ); + } + (retry, result) => { + return Err(format!("prefix {count}: expected {retry:?}, got {result:?}").into()); + } + } + } + Ok(()) +} + +#[test] +fn a_crash_during_each_stage_write_discards_only_that_stage() -> Result<(), Box> { + for (phase, stage, discard) in [ + (2, "root.next", Step::DiscardRootStage), + (8, "manifest.next", Step::DiscardManifestStage), + (12, "head.next", Step::DiscardHeadStage), + ] { + let name = format!("filesystem-retention-recovery-during-{phase}"); + let (sandbox, mut authority) = open_authority(&name)?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + drive_publication(&mut authority, &preparation, phase - 1)?; + let publication = preparation.publication().ok_or("no publication")?; + let complete: &[u8] = match phase { + 2 => preparation.candidate().encoded(), + 8 => publication.manifest().encoded(), + _ => publication.head().encoded(), + }; + // 100 bytes is inside every record's framing: the head is 144 bytes, the + // manifest header 160, and the root header 192. + let partial = complete.get(..100).ok_or("record shorter than 100 bytes")?; + fs::write(sandbox.path().join("retention").join(stage), partial)?; + let (mut steps, outcome, _retry) = expected(phase - 1); + steps.insert(0, discard); + + let receipt = authority + .recover() + .map_err(|error| format!("during {phase}: {error}"))?; + + assert_eq!(receipt.executed(), steps, "during {phase}: steps"); + assert_eq!(receipt.outcome(), outcome, "during {phase}: outcome"); + assert!(!sandbox.path().join("retention").join(stage).exists()); + } + Ok(()) +} + +#[test] +fn successor_prefixes_recover_against_the_published_generation() -> Result<(), Box> { + for count in [2, 9, 13, 15] { + let name = format!("filesystem-retention-recovery-successor-{count}"); + let (_sandbox, mut authority) = open_authority(&name)?; + let root_bytes = fixture(ROOT_HEX)?; + let _published = + execute_retention_publication(&mut authority, &initial_preparation(&root_bytes)?)?; + let current_root = AdmittedRetentionRoot::decode(&root_bytes)?; + let manifest_bytes = fixture(MANIFEST_HEX)?; + let current_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let candidate = successor_root(¤t_root)?; + let preparation = + successor_preparation(¤t_root, ¤t_manifest, candidate.encoded())?; + drive_publication(&mut authority, &preparation, count)?; + let (steps, outcome, _retry) = expected(count); + + let receipt = authority + .recover() + .map_err(|error| format!("successor {count}: {error}"))?; + + assert_eq!(receipt.executed(), steps, "successor {count}: steps"); + assert_eq!(receipt.outcome(), outcome, "successor {count}: outcome"); + if outcome == Outcome::Committed { + let observed = authority.observe_current()?.ok_or("no head after commit")?; + assert_eq!( + observed.head().generation(), + preparation.liveness_generation() + ); + } + } + Ok(()) +} diff --git a/src/adapters/retention/filesystem_retention_recovery_tests.rs b/src/adapters/retention/filesystem_retention_recovery_tests.rs index 0cb5d3a..798676c 100644 --- a/src/adapters/retention/filesystem_retention_recovery_tests.rs +++ b/src/adapters/retention/filesystem_retention_recovery_tests.rs @@ -4,50 +4,14 @@ use std::error::Error; use std::fs; use super::filesystem_retention_test_fixture::{ - ROOT_HEX, fixture, head_path, initial_preparation, manifest_pool_path, open_authority, - root_pool_path, + ROOT_HEX, drive_publication, fixture, head_path, initial_preparation, manifest_pool_path, + open_authority, root_pool_path, }; use super::{ - FilesystemRetentionPublicationAuthority, RetentionPublicationOutcome, - RetentionPublicationPreparation, RetentionPublicationStorage, - RetentionRecoveryOutcome as Outcome, RetentionRecoveryStep as Step, + RetentionPublicationOutcome, RetentionRecoveryOutcome as Outcome, RetentionRecoveryStep as Step, }; use crate::execute_retention_publication; -type Phase<'a> = - &'a mut dyn FnMut(&mut FilesystemRetentionPublicationAuthority) -> std::io::Result<()>; - -/// Executes publication phases 1 through `count` and stops, like a crash there. -fn drive( - authority: &mut FilesystemRetentionPublicationAuthority, - preparation: &RetentionPublicationPreparation<'_>, - count: usize, -) -> Result<(), Box> { - let publication = preparation - .publication() - .ok_or("preparation carries no publication")?; - let root = preparation.candidate(); - let phases: [Phase<'_>; 13] = [ - &mut |a| a.verify_current(preparation).map(|_| ()), - &mut |a| a.write_root_stage(root), - &mut |a| a.synchronize_root_stage(), - &mut |a| a.admit_root_namespace(root).map(|_| ()), - &mut |a| a.synchronize_roots_after_namespace(), - &mut |a| a.link_root(root), - &mut |a| a.synchronize_root_namespace(root), - &mut |a| a.write_manifest_stage(publication.manifest()), - &mut |a| a.synchronize_manifest_stage(), - &mut |a| a.link_manifest(publication.manifest()), - &mut |a| a.synchronize_manifest_pool(), - &mut |a| a.write_head_stage(publication.head()), - &mut |a| a.synchronize_head_stage(), - ]; - for phase in phases.into_iter().take(count) { - phase(authority)?; - } - Ok(()) -} - #[test] fn a_clean_store_recovers_to_clean() -> Result<(), Box> { let (_sandbox, mut authority) = open_authority("filesystem-retention-recovery-clean")?; @@ -62,7 +26,7 @@ fn a_written_root_stage_is_linked_and_protected() -> Result<(), Box> let (sandbox, mut authority) = open_authority("filesystem-retention-recovery-root")?; let root_bytes = fixture(ROOT_HEX)?; let preparation = initial_preparation(&root_bytes)?; - drive(&mut authority, &preparation, 3)?; + drive_publication(&mut authority, &preparation, 3)?; let receipt = authority.recover()?; @@ -89,7 +53,7 @@ fn a_synchronized_head_stage_is_finalized_and_the_retry_is_already_committed() let (sandbox, mut authority) = open_authority("filesystem-retention-recovery-head")?; let root_bytes = fixture(ROOT_HEX)?; let preparation = initial_preparation(&root_bytes)?; - drive(&mut authority, &preparation, 13)?; + drive_publication(&mut authority, &preparation, 13)?; let publication = preparation.publication().ok_or("no publication")?; let receipt = authority.recover()?; diff --git a/src/adapters/retention/filesystem_retention_test_fixture.rs b/src/adapters/retention/filesystem_retention_test_fixture.rs index 9672c10..988e567 100644 --- a/src/adapters/retention/filesystem_retention_test_fixture.rs +++ b/src/adapters/retention/filesystem_retention_test_fixture.rs @@ -8,6 +8,7 @@ use std::fs; use std::io; use std::path::{Path, PathBuf}; +use super::RetentionPublicationStorage; use super::filesystem_retention_authority::FilesystemRetentionPublicationAuthority; use super::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionRoot, @@ -261,3 +262,49 @@ fn write_version_one(sandbox: &TestDirectory) -> Result<(), Box> { const fn maximum_policy() -> SegmentReadPolicy { SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) } + +type PublicationPhase<'a> = + &'a mut dyn FnMut(&mut FilesystemRetentionPublicationAuthority) -> io::Result<()>; + +/// The number of storage-port phases one publication executes. +pub(super) const PUBLICATION_PHASE_COUNT: usize = 18; + +/// Executes publication phases 1 through `count` and stops, like a crash there. +/// +/// Phase 1 is current-state verification; 2 through 18 are the storage-port +/// phases in `RetentionPublicationPhase::ALL` order, so `count` selects the +/// exact prefix a process death after that phase would leave behind. +pub(super) fn drive_publication( + authority: &mut FilesystemRetentionPublicationAuthority, + preparation: &RetentionPublicationPreparation<'_>, + count: usize, +) -> Result<(), Box> { + let publication = preparation + .publication() + .ok_or("preparation carries no publication")?; + let root = preparation.candidate(); + let phases: [PublicationPhase<'_>; PUBLICATION_PHASE_COUNT] = [ + &mut |a| a.verify_current(preparation).map(|_| ()), + &mut |a| a.write_root_stage(root), + &mut |a| a.synchronize_root_stage(), + &mut |a| a.admit_root_namespace(root).map(|_| ()), + &mut |a| a.synchronize_roots_after_namespace(), + &mut |a| a.link_root(root), + &mut |a| a.synchronize_root_namespace(root), + &mut |a| a.write_manifest_stage(publication.manifest()), + &mut |a| a.synchronize_manifest_stage(), + &mut |a| a.link_manifest(publication.manifest()), + &mut |a| a.synchronize_manifest_pool(), + &mut |a| a.write_head_stage(publication.head()), + &mut |a| a.synchronize_head_stage(), + &mut |a| a.replace_head(), + &mut |a| a.synchronize_retention_namespace(), + &mut |a| a.remove_root_stage(), + &mut |a| a.remove_manifest_stage(), + &mut |a| a.synchronize_cleanup(), + ]; + for phase in phases.into_iter().take(count) { + phase(authority)?; + } + Ok(()) +} From 3d031b4b601bef0245bb8db88937cce91085242b Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 9 Sep 2026 13:30:19 -0700 Subject: [PATCH 5/8] Fix: run retention recovery as the first step of publication The retention publication page has always listed "completes recovery of every fixed retention stage" as publication's first step, and until now the filesystem writer refused every retained stage instead, which left an interrupted publication waiting for a human. verify_current now calls recover before anything else. A clean or committed outcome continues; a protected outcome (complete orphans awaiting explicit disposition) refuses RetainedStage as before; recovery's planning refusal and step failure travel as RecoveryRefused { source } and RecoveryStepRefused { source } through RetentionCurrentStateRefusal, so callers keep recovery's own reason. Three laws that pinned the refuse-everything doctrine now pin the recovered behaviour: a truncated manifest stage is discarded and publication publishes (this law failed before the change), a complete orphan root stage still refuses and stays retained and linked, and a complete head stage without its manifest refuses with recovery's ambiguity. README, the version-two overview, the retention page, and the ledger nonclaims describe the recovered behaviour and name the two remaining waits: complete orphans until disposition (#21) and process-death evidence (#19). Refs #19 #21 --- CHANGELOG.md | 5 +- README.md | 14 ++++-- docs/formats/segment-store-v2/README.md | 8 +-- docs/formats/segment-store-v2/requirements.md | 9 ++-- docs/formats/segment-store-v2/retention.md | 5 +- .../filesystem_retention_attempt_tests.rs | 6 +-- .../filesystem_retention_recovery.rs | 4 +- .../retention/filesystem_retention_refusal.rs | 17 +++++++ .../retention/filesystem_retention_storage.rs | 20 +++++++- .../filesystem_retention_storage_tests.rs | 49 +++++++++++++++---- 10 files changed, 104 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e7cb39..9bc32b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,10 @@ after its public API and format compatibility policies are established. publication prefix from 0 through 18 phases, truncate each stage mid-write, and replay successor prefixes over a published generation; each recovers to its documented state, recovery is idempotent, and the forward retry reports - the predicted outcome. + the predicted outcome. Publication runs that recovery as its first step, so + an interrupted publication no longer waits for a human unless it left a + complete orphan; `RecoveryRefused` and `RecoveryStepRefused` carry + recovery's own errors through `RetentionCurrentStateRefusal`. - `FilesystemRetentionPublicationAuthority` executes the 17 ordered retention publication phases against a completely migrated version-2 root. It stages `root.next`, `manifest.next`, and `head.next` exclusively, verifies device diff --git a/README.md b/README.md index 51b601c..9453d5a 100644 --- a/README.md +++ b/README.md @@ -69,11 +69,15 @@ Keep is required to refuse all three, before mutating anything. ## What it does not do yet -Version 2 writes correctly from a clean start and, if it finds the residue of -an interrupted publication, refuses rather than guesses. Nothing yet recovers -that residue, and readers have no fence, so **an interrupted version-2 -publication waits for a human until #19 lands.** A version-1 store stays -admitted until its owner migrates it; migrate only if you accept that wait. +Version 2 writes correctly from a clean start, and the next publication +recovers the residue of an interrupted one: a stage cut mid-write is +discarded, a head already synchronized is finalized, and a byte-identical +retry reports already committed. The one state that waits for a human is a +complete orphan, a crash between the root link and the head finalization, +which stays recovery-protected until explicit disposition lands with garbage +collection (#21). Readers have no fence yet, and process-death evidence for +the recovery itself is still to come (#19). A version-1 store stays admitted +until its owner migrates it. | Gap | Tracked | | --- | --- | diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 1f27b80..20b19f2 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -94,10 +94,10 @@ head and the catalog it selects, and refuses superseded candidates, retained stages, replaced protocol directories, and every namespace or capacity violation before mutation, each as a typed `RetentionCurrentStateRefusal`. -Not implemented: retention publication recovery and `KEEP-CRASH-036..052` -process-death evidence, partial-prefix migration recovery and -`KEEP-CRASH-053..073`, the reader fence, model-based transition evidence, and -garbage collection. Issue #19 owns the first four and issue #21 the last; +Retention publication recovery is implemented and proven in-process for every +crash prefix; its `KEEP-CRASH-036..052` process-death evidence is not. +Not implemented: partial-prefix migration recovery and `KEEP-CRASH-053..073`, +the reader fence, model-based transition evidence, and garbage collection. Issue #19 owns the first four and issue #21 the last; issue #97 owns the restart-stable root identity coordinate. A version-1 store remains admitted until its owner migrates it, and the [requirements ledger](requirements.md) is the authority on which requirements diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 1fbda2d..b604f9e 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -60,10 +60,11 @@ case is not evidence. - A fresh forward writer is not proof that version 2 is restart-safe or production-admitted; partial-prefix recovery and crash evidence remain mandatory. This applies to retention publication exactly as it applies to - migration: the filesystem publication writer refuses every retained stage - instead of continuing it. A stage left behind by a failed write is recovery - evidence like any crash residue; it is never unlinked, and the next - publication refuses until recovery classifies it. + migration: the filesystem publication writer never continues a retained + stage; it recovers it first, discarding a pre-effect truncated stage, + finalizing a complete head, and refusing a complete orphan until explicit + disposition. A stage left behind by a failed write is recovery evidence like + any crash residue and is classified the same way. - Publication binds this store's catalog `HEAD` to the verified closure and reopens the head-selected catalog pool entry under authority, but it does not re-read closure-member segments: every read authenticates them, and diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index 6f7402f..44a8668 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -165,8 +165,9 @@ implements root, manifest, and head codecs with a typed verified anchor-set digest, expected-state transition planning, deterministic closure verification, a blocking publication storage capability port, and ordered storage-port orchestration. `FilesystemRetentionPublicationAuthority` publishes initial and -successor generations against its observed head and refuses superseded -candidates and retained stages; recovery, fencing, and collection remain absent. +successor generations against its observed head, recovers retained stages +first, and refuses superseded candidates and protected orphans; fencing and +collection remain absent. ## Global retention manifest diff --git a/src/adapters/retention/filesystem_retention_attempt_tests.rs b/src/adapters/retention/filesystem_retention_attempt_tests.rs index 01c5cec..c24aa30 100644 --- a/src/adapters/retention/filesystem_retention_attempt_tests.rs +++ b/src/adapters/retention/filesystem_retention_attempt_tests.rs @@ -20,17 +20,17 @@ fn refused_verification_admits_no_later_phase() -> Result<(), Box> { let preparation = initial_preparation(&root_bytes)?; fs::write( sandbox.path().join("retention").join("head.next"), - b"retained", + fixture(super::filesystem_retention_test_fixture::HEAD_HEX)?, )?; let before = retention_witness(sandbox.path())?; let error = authority .verify_current(&preparation) .err() - .ok_or("retained head stage was admitted")?; + .ok_or("an ambiguous head stage was admitted")?; assert!(matches!( refusal(&error), - Some(RetentionCurrentStateRefusal::RetainedStage) + Some(RetentionCurrentStateRefusal::RecoveryRefused { .. }) )); let error = authority .write_root_stage(preparation.candidate()) diff --git a/src/adapters/retention/filesystem_retention_recovery.rs b/src/adapters/retention/filesystem_retention_recovery.rs index 1de0e93..d350d8e 100644 --- a/src/adapters/retention/filesystem_retention_recovery.rs +++ b/src/adapters/retention/filesystem_retention_recovery.rs @@ -117,8 +117,8 @@ impl FilesystemRetentionPublicationAuthority { /// returns an empty receipt; a truncated pre-effect stage is discarded; a /// complete stage is linked and retained as a recovery-protected orphan; /// a complete head over linked stages is finalized. Any pending - /// publication attempt is discarded first, and the caller re-verifies - /// current state afterwards. + /// publication attempt is discarded first. Publication calls this itself + /// as its first step; callers may also run it explicitly at restart. /// /// # Errors /// diff --git a/src/adapters/retention/filesystem_retention_refusal.rs b/src/adapters/retention/filesystem_retention_refusal.rs index be4d20d..598565c 100644 --- a/src/adapters/retention/filesystem_retention_refusal.rs +++ b/src/adapters/retention/filesystem_retention_refusal.rs @@ -5,6 +5,7 @@ use std::fmt; use std::io; use super::{RetentionHeadDecodeError, RetentionManifestDecodeError}; +use super::{RetentionRecoveryError, RetentionRecoveryRefusal}; use crate::adapters::{CatalogDecodeError, PublicationHeadDecodeError}; use crate::{CatalogGeneration, LivenessGeneration, RetentionManifestDigest}; @@ -122,6 +123,16 @@ pub enum RetentionCurrentStateRefusal { /// A protocol directory named at admission (`retention`, `roots`, or /// `manifests`) no longer names the pinned directory that was admitted. ProtocolDirectoryReplaced, + /// Restart recovery refused the retained stages as unrecoverable ambiguity. + RecoveryRefused { + /// The exact planning refusal. + source: RetentionRecoveryRefusal, + }, + /// A restart recovery step refused; the completed prefix remains. + RecoveryStepRefused { + /// The refused step, the completed prefix, and the storage error. + source: RetentionRecoveryError, + }, /// A record's kind or length disagreed with its declaration. RecordKindOrLength, /// A record carried bytes beyond its declared length. @@ -228,6 +239,10 @@ impl RetentionCurrentStateRefusal { Self::RecordKindOrLength => "retention record kind or length disagreed", Self::RecordTrailingBytes => "retention record carried trailing bytes", Self::RecordLengthOverflow => "retention record length exceeded the addressable range", + Self::RecoveryRefused { .. } => { + "restart recovery refused the retained retention stages" + } + Self::RecoveryStepRefused { .. } => "a restart recovery step refused", Self::ProtocolDirectoryReplaced => { "a retention protocol directory was replaced after admission" } @@ -253,6 +268,8 @@ impl Error for RetentionCurrentStateRefusal { Self::ManifestRefused { source } => Some(source), Self::CatalogHeadRefused { source } => Some(source), Self::CatalogRefused { source } => Some(source.as_ref()), + Self::RecoveryRefused { source } => Some(source), + Self::RecoveryStepRefused { source } => Some(source), _ => None, } } diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs index e5c897c..ac943ec 100644 --- a/src/adapters/retention/filesystem_retention_storage.rs +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -14,8 +14,9 @@ use super::filesystem_retention_pool_name as pool_name; use super::filesystem_retention_stage::FilesystemRetentionStage; use super::{ AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, - RetentionCurrentStateRefusal, RetentionNamespaceAdmission, RetentionPublicationPreparation, - RetentionPublicationStorage, RetentionTransitionDisposition, + FilesystemRetentionRecoveryError, RetentionCurrentStateRefusal, RetentionNamespaceAdmission, + RetentionPublicationPreparation, RetentionPublicationStorage, RetentionRecoveryOutcome, + RetentionTransitionDisposition, }; use crate::RetentionGenerationExpectation; use crate::adapters::filesystem_catalog_artifact::synchronize_directory; @@ -28,6 +29,21 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { ) -> io::Result { self.attempt = None; require_pinned_directories(&self.root, &self.retention, &self.roots, &self.manifests)?; + let recovery = self.recover().map_err(|error| match error { + FilesystemRetentionRecoveryError::Observe { source } => source, + FilesystemRetentionRecoveryError::Plan { source } => { + RetentionCurrentStateRefusal::RecoveryRefused { source }.into_io() + } + FilesystemRetentionRecoveryError::Execute { source } => { + RetentionCurrentStateRefusal::RecoveryStepRefused { source }.into_io() + } + })?; + if matches!( + recovery.outcome(), + RetentionRecoveryOutcome::Protected { .. } + ) { + return Err(RetentionCurrentStateRefusal::RetainedStage.into_io()); + } require_no_retained_stage(&self.retention)?; let census = filesystem_retention_namespace::admit(&self.retention, &self.roots, &self.manifests)?; diff --git a/src/adapters/retention/filesystem_retention_storage_tests.rs b/src/adapters/retention/filesystem_retention_storage_tests.rs index 3c13257..697fd55 100644 --- a/src/adapters/retention/filesystem_retention_storage_tests.rs +++ b/src/adapters/retention/filesystem_retention_storage_tests.rs @@ -77,6 +77,10 @@ fn retained_stage_refuses_publication_before_recovery() -> Result<(), Box io::Result)>> { } #[test] -fn retained_manifest_stage_refuses_publication_before_recovery() -> Result<(), Box> { - let (sandbox, mut authority) = - open_authority("filesystem-retention-recovery-required-manifest")?; +fn a_complete_orphan_root_stage_refuses_publication_until_disposition() -> Result<(), Box> +{ + let (sandbox, mut authority) = open_authority("filesystem-retention-recovery-required-orphan")?; let root_bytes = fixture(ROOT_HEX)?; let preparation = initial_preparation(&root_bytes)?; fs::write( - sandbox.path().join("retention").join("manifest.next"), - b"partial bytes left by a failed write", + sandbox.path().join("retention").join("root.next"), + &root_bytes, )?; - let before = retention_witness(sandbox.path())?; let error = execute_retention_publication(&mut authority, &preparation) .err() - .ok_or("retained manifest stage was unexpectedly published over")?; + .ok_or("a complete orphan root stage was unexpectedly published over")?; let RetentionPublicationError::CurrentVerification { source } = error else { - return Err("retained stage refused outside current-state verification".into()); + return Err("protected orphan refused outside current-state verification".into()); }; - assert_eq!(source.kind(), io::ErrorKind::InvalidData); - assert_eq!(retention_witness(sandbox.path())?, before); + assert!(matches!( + super::filesystem_retention_test_fixture::refusal(&source), + Some(super::RetentionCurrentStateRefusal::RetainedStage) + )); + assert!(sandbox.path().join("retention").join("root.next").is_file()); + assert_eq!( + fs::read(root_pool_path(sandbox.path(), preparation.candidate()))?, + root_bytes + ); + Ok(()) +} + +#[test] +fn a_truncated_stage_is_recovered_and_publication_proceeds() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-recovered-stage")?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + let stage = sandbox.path().join("retention").join("manifest.next"); + fs::write(&stage, b"partial bytes left by a failed write")?; + + let receipt = execute_retention_publication(&mut authority, &preparation)?; + + assert_eq!(receipt.outcome(), RetentionPublicationOutcome::Published); + assert!( + !stage.exists(), + "the truncated stage must be discarded by recovery" + ); + assert_eq!(fs::read(head_path(sandbox.path()))?, fixture(HEAD_HEX)?); Ok(()) } From 7a512c179d41a8deb4f486d601e25b91d51116f2 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 9 Sep 2026 14:04:12 -0700 Subject: [PATCH 6/8] Add: kill real writers at every retention publication coordinate The durability crash matrix now covers KEEP-CRASH-036 through 052. A child initializes a store, writes the golden bundle corpus, migrates it through all 21 phases, reopens it as version two, prepares retention generation one against the bundle catalog snapshot, and publishes through a decorator that dies before, during, or after the selected phase. During a stage write the decorator leaves a 100-byte prefix, inside every record's framing, so restart classifies it as truncated rather than corrupt. Restart reopens the store through the same admission a production caller would use, runs FilesystemRetentionPublicationAuthority::recover, and requires the documented steps and outcome for that exact prefix, then requires the forward retry to report what recovery predicts: published after a clean prefix, refused as recovery-required while protected orphans remain, already committed once the head is finalized. All 51 retention coordinates pass, and the complete 156-case matrix passes locally. FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks and FilesystemStoreMigrationAuthority::open_unchecked_for_repository_tasks give repository tools the bypass version one already had; every namespace, record, and identity law still applies through them. KEEP-RETENTION-007 is now Implemented in the ledger; the README, overview, and recovery page say that process-death evidence exists. Refs #19 --- CHANGELOG.md | 9 +- README.md | 14 +- docs/formats/segment-store-v2/README.md | 5 +- docs/formats/segment-store-v2/recovery.md | 5 +- docs/formats/segment-store-v2/requirements.md | 2 +- .../filesystem_version_two_admission.rs | 21 +- .../filesystem_migration_authority.rs | 20 ++ .../production_protocol.rs | 5 + .../production_protocol/fixture.rs | 34 +++ .../production_protocol/initialization.rs | 17 +- .../production_protocol/retention.rs | 114 +++++++++ .../production_protocol/retention_storage.rs | 229 ++++++++++++++++++ xtask/src/durability_crash_matrix/restart.rs | 4 + .../restart/expectation.rs | 5 + .../restart/retention.rs | 162 +++++++++++++ xtask/src/durability_crash_point.rs | 72 +++++- xtask/src/durability_crash_point_identity.rs | 17 ++ .../tests/durability_crash_point_contract.rs | 83 ++++++- 18 files changed, 797 insertions(+), 21 deletions(-) create mode 100644 xtask/src/durability_crash_matrix/production_protocol/retention.rs create mode 100644 xtask/src/durability_crash_matrix/production_protocol/retention_storage.rs create mode 100644 xtask/src/durability_crash_matrix/restart/retention.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bc32b5..93f740f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,14 @@ after its public API and format compatibility policies are established. the predicted outcome. Publication runs that recovery as its first step, so an interrupted publication no longer waits for a human unless it left a complete orphan; `RecoveryRefused` and `RecoveryStepRefused` carry - recovery's own errors through `RetentionCurrentStateRefusal`. + recovery's own errors through `RetentionCurrentStateRefusal`. The crash + matrix gains `KEEP-CRASH-036` through `052`: a child migrates a golden + bundle store, publishes retention generation one, and is killed before, + during, or after each of the seventeen phases; restart reopens the store, + runs recovery, and requires the documented steps, outcome, and forward + retry. `FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks` + and `FilesystemStoreMigrationAuthority::open_unchecked_for_repository_tasks` + give repository tools the same bypass version one already had. - `FilesystemRetentionPublicationAuthority` executes the 17 ordered retention publication phases against a completely migrated version-2 root. It stages `root.next`, `manifest.next`, and `head.next` exclusively, verifies device diff --git a/README.md b/README.md index 9453d5a..cedc6fa 100644 --- a/README.md +++ b/README.md @@ -51,10 +51,10 @@ Keep is required to refuse all three, before mutating anything. generation-versioned catalogs, and a fixed-width `HEAD` are published through an ordered protocol whose every step is a named crash point. Platform admission is Linux ext4, non-casefolded, one writer. -- **Proven restart recovery for version 1.** The crash matrix kills real - writer processes at 105 before/during/after coordinates - (`KEEP-CRASH-001`–`035`) and verifies the store lands in exactly one - documented lawful state each time. +- **Proven restart recovery.** The crash matrix kills real writer processes + at 156 before/during/after coordinates (`KEEP-CRASH-001`–`052`) and + verifies the store lands in exactly one documented lawful state each time, + for version-1 publication and for version-2 retention publication. - **Version-2 retention and migration, forward path.** Explicit retention roots, deterministic closure verification, a one-way 21-phase migration, and a 17-phase retention publication — all with production filesystem @@ -75,9 +75,9 @@ discarded, a head already synchronized is finalized, and a byte-identical retry reports already committed. The one state that waits for a human is a complete orphan, a crash between the root link and the head finalization, which stays recovery-protected until explicit disposition lands with garbage -collection (#21). Readers have no fence yet, and process-death evidence for -the recovery itself is still to come (#19). A version-1 store stays admitted -until its owner migrates it. +collection (#21). The crash matrix proves that recovery by killing real +writer processes at all 51 retention coordinates. Readers have no fence yet +(#19). A version-1 store stays admitted until its owner migrates it. | Gap | Tracked | | --- | --- | diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 20b19f2..e3a8274 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -94,8 +94,9 @@ head and the catalog it selects, and refuses superseded candidates, retained stages, replaced protocol directories, and every namespace or capacity violation before mutation, each as a typed `RetentionCurrentStateRefusal`. -Retention publication recovery is implemented and proven in-process for every -crash prefix; its `KEEP-CRASH-036..052` process-death evidence is not. +Retention publication recovery is implemented and proven both in-process for +every crash prefix and by the crash matrix, which kills a real writer before, +during, and after `KEEP-CRASH-036` through `052`. Not implemented: partial-prefix migration recovery and `KEEP-CRASH-053..073`, the reader fence, model-based transition evidence, and garbage collection. Issue #19 owns the first four and issue #21 the last; issue #97 owns the restart-stable root identity coordinate. A version-1 store diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index d4449ad..15049a1 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -233,8 +233,9 @@ The retention crash points are: `RetentionPublicationPhase::ALL` freezes this exact order as a typed public vocabulary. `FilesystemRetentionPublicationAuthority::recover` implements the -classification above and its effects; process-death evidence remains -unimplemented. +classification above and its effects, and the crash matrix kills a real +writer before, during, and after every point and requires restart to recover +to the documented state. Each point requires before, during, and after process-death evidence. Restart must establish exact catalog visibility, retention head, namespace generation, diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index b604f9e..6d1d318 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -15,7 +15,7 @@ case is not evidence. | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; writer-locked initial filesystem publication in `filesystem_retention_storage_tests`; observed-head successor publication, exact predecessor binding, and absent-head refusal in `filesystem_retention_successor_tests`; the store's catalog head must name the closure's catalog generation and digest before any forward write in `filesystem_retention_catalog_tests`; a head whose predecessor disagrees with its manifest refuses in `filesystem_retention_current_tests`; a successor reopens and decodes the manifest-selected predecessor root and refuses an absent or changed one in `filesystem_retention_expectation_tests` | Implemented | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; production 17-phase forward filesystem execution, exclusive staging, byte-equal inode-substitution refusal, and retained-stage recovery refusal in `filesystem_retention_storage_tests`; orphan namespace directories count against the 4,096 ceiling and refuse a new namespace before any stage is written in `filesystem_retention_capacity_tests`; crash injection remains | In progress in #19 | -| `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | recovery-required refusals before any mutation in `filesystem_retention_expectation_tests`: an absent head over populated pools, a non-initial head prepared against an absent head, an orphan directory for a namespace expected absent, and an absent directory for a namespace expected current; debug and release crash matrix remains; replaced protocol directories, an absent or changed head-selected catalog, an over-full census, zero-generation pool names, and a stage retained by a failed write refuse in `filesystem_retention_*_tests`; storage-independent classification of every fixed-stage crash prefix (discard, link and protect, finalize, clean up, or typed refusal) in `recovery_planner_tests`; every publication prefix 0 through 18, each mid-write truncation, and successor prefixes recover in-process to the documented state, idempotently, with the forward retry reporting the predicted outcome, in `filesystem_retention_recovery_prefix_tests` | In progress in #19 | +| `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | recovery-required refusals before any mutation in `filesystem_retention_expectation_tests`: an absent head over populated pools, a non-initial head prepared against an absent head, an orphan directory for a namespace expected absent, and an absent directory for a namespace expected current; replaced protocol directories, an absent or changed head-selected catalog, an over-full census, zero-generation pool names, and a stage retained by a failed write refuse in `filesystem_retention_*_tests`; storage-independent classification of every fixed-stage crash prefix (discard, link and protect, finalize, clean up, or typed refusal) in `recovery_planner_tests`; every publication prefix 0 through 18, each mid-write truncation, and successor prefixes recover in-process to the documented state, idempotently, with the forward retry reporting the predicted outcome, in `filesystem_retention_recovery_prefix_tests`; `cargo xtask durability-crash-matrix` kills a real writer before, during, and after `KEEP-CRASH-036` through `052` and requires restart recovery to reach the documented state and the forward retry to report the predicted outcome | Implemented | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | | `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests`; committed retry reopens the head-selected manifest entry and root pool bytes, refusing absent, changed, or corrupt evidence in `filesystem_retention_current_tests`; every refusal is a typed `RetentionCurrentStateRefusal` source, with superseded, committed-root-absent, committed-root-changed, and head-absent-with-artifacts pinned by downcast | Implemented | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | model-based and source-architecture tests | Planned in #19 | diff --git a/src/adapters/filesystem_version_two_admission.rs b/src/adapters/filesystem_version_two_admission.rs index 2303c5a..5d03199 100644 --- a/src/adapters/filesystem_version_two_admission.rs +++ b/src/adapters/filesystem_version_two_admission.rs @@ -3,7 +3,7 @@ use std::path::Path; use cap_fs_ext::DirExt; -#[cfg(test)] +#[cfg(any(test, feature = "repository-tasks"))] use cap_std::ambient_authority; use cap_std::fs::Dir; @@ -64,6 +64,25 @@ impl FilesystemVersionTwoAdmission { } /// Releases the writer lock and the three pinned retention capabilities. + /// Reopens a migrated root without platform admission for repository tasks. + /// + /// The crash matrix and other repository tools run on hosts outside the + /// admitted Linux profile; every namespace, record, and identity law still + /// applies. Production callers use [`Self::reopen`]. + /// + /// # Errors + /// + /// Returns [`FilesystemPlatformAdmissionError`] exactly as [`Self::reopen`] + /// does for every boundary after platform admission. + #[cfg(feature = "repository-tasks")] + pub fn reopen_unchecked_for_repository_tasks( + store_root: &Path, + ) -> Result { + let root = Dir::open_ambient_dir(store_root, ambient_authority()) + .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + Self::admit(root) + } + pub(super) fn into_parts(self) -> (FilesystemWriterLock, Dir, Dir, Dir) { (self.lock, self.retention, self.roots, self.manifests) } diff --git a/src/adapters/store_migration/filesystem_migration_authority.rs b/src/adapters/store_migration/filesystem_migration_authority.rs index e894bcb..2277bfd 100644 --- a/src/adapters/store_migration/filesystem_migration_authority.rs +++ b/src/adapters/store_migration/filesystem_migration_authority.rs @@ -70,6 +70,26 @@ impl FilesystemStoreMigrationAuthority { }) } + /// Pins a root for migration without platform admission for repository tasks. + /// + /// Repository tools such as the crash matrix run on hosts outside the + /// admitted Linux profile; namespace, head, catalog, inventory, and record + /// laws still apply in full. Production callers use [`Self::open`]. + /// + /// # Errors + /// + /// Returns [`FilesystemMigrationAuthorityError`](super::FilesystemMigrationAuthorityError) + /// when the root identity cannot be read or the pools cannot be pinned. + #[cfg(feature = "repository-tasks")] + pub fn open_unchecked_for_repository_tasks( + lock: crate::adapters::FilesystemWriterLock, + policy: SegmentReadPolicy, + ) -> Result { + let admission = FilesystemPlatformAdmission::unchecked_for_repository_tasks(lock) + .map_err(|source| Error::RootIdentity { source })?; + Self::open(admission, policy) + } + /// Observes one canonical intent from exact current version-1 authority. /// /// The synchronous call admits the exact published root namespace, physical diff --git a/xtask/src/durability_crash_matrix/production_protocol.rs b/xtask/src/durability_crash_matrix/production_protocol.rs index 6453a94..fa842b2 100644 --- a/xtask/src/durability_crash_matrix/production_protocol.rs +++ b/xtask/src/durability_crash_matrix/production_protocol.rs @@ -8,6 +8,8 @@ mod publication; mod publication_storage; mod recovery; mod recovery_storage; +pub(super) mod retention; +mod retention_storage; mod segment_stage; use std::error::Error; @@ -41,6 +43,9 @@ pub(super) fn run( DurabilityCrashSequence::RecoveryDiscard => { recovery::run(&store_root, &mut control)?; } + DurabilityCrashSequence::Retention => { + retention::run(&store_root, &mut control)?; + } } Err(DurabilityCrashMatrixError::PointSequenceMismatch { point: case.point(), diff --git a/xtask/src/durability_crash_matrix/production_protocol/fixture.rs b/xtask/src/durability_crash_matrix/production_protocol/fixture.rs index 728f7dd..4f6f1b0 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/fixture.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/fixture.rs @@ -11,6 +11,20 @@ const SEGMENT_HEX: &str = const CATALOG_HEX: &str = include_str!("../../../../conformance/segment-store/v1/one-zero-catalog.hex"); const HEAD_HEX: &str = include_str!("../../../../conformance/segment-store/v1/one-zero-head.hex"); +const BUNDLE_SEGMENT_HEX: &str = + include_str!("../../../../conformance/segment-store/v1/one-zero-bundle-segment.hex"); +const BUNDLE_CATALOG_HEX: &str = + include_str!("../../../../conformance/segment-store/v1/one-zero-bundle-catalog.hex"); +const BUNDLE_HEAD_HEX: &str = + include_str!("../../../../conformance/segment-store/v1/one-zero-bundle-head.hex"); +const RETENTION_ROOT_HEX: &str = + include_str!("../../../../conformance/segment-store/v2/one-anchor-root.hex"); +/// Pool name of the bundle segment the retention root's closure references. +pub(in crate::durability_crash_matrix) const BUNDLE_SEGMENT_NAME: &str = + "221f6745cd8a5221c9a87c3707593608479282b54a4a74d0e753fd76f70e8db2.seg"; +/// Pool name of the generation-one bundle catalog. +pub(in crate::durability_crash_matrix) const BUNDLE_CATALOG_NAME: &str = + "0000000000000001-0b7cad1b6de663d34beacbc214db7497f2e36ab6b08dfbd5febbc8d06a418811.cat"; pub(in crate::durability_crash_matrix) const SEGMENT_POOL_PATH: &str = "segments/b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc.seg"; @@ -31,6 +45,26 @@ impl GoldenFixture { Self::decode("catalog", CATALOG_HEX, 352) } + pub(in crate::durability_crash_matrix) fn bundle_segment() + -> Result { + Self::decode("bundle segment", BUNDLE_SEGMENT_HEX, 701) + } + + pub(in crate::durability_crash_matrix) fn bundle_catalog() + -> Result { + Self::decode("bundle catalog", BUNDLE_CATALOG_HEX, 512) + } + + pub(in crate::durability_crash_matrix) fn bundle_head() + -> Result { + Self::decode("bundle head", BUNDLE_HEAD_HEX, 128) + } + + pub(in crate::durability_crash_matrix) fn retention_root() + -> Result { + Self::decode("retention root", RETENTION_ROOT_HEX, 378) + } + pub(in crate::durability_crash_matrix) fn head() -> Result { Self::decode("head", HEAD_HEX, 128) } diff --git a/xtask/src/durability_crash_matrix/production_protocol/initialization.rs b/xtask/src/durability_crash_matrix/production_protocol/initialization.rs index 6bd5466..9770173 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/initialization.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/initialization.rs @@ -25,16 +25,23 @@ pub(super) fn run( .map_err(|source| verification("execute production store initialization", source)) } -pub(super) fn publisher( +/// Initializes a fresh store and returns its retained writer lock. +pub(super) fn initialized_lock( store_root: &Path, -) -> Result { +) -> Result { let mut storage = RepositoryInitializationStorage::admit_unchecked(store_root) .map_err(|source| DurabilityCrashMatrixError::io("open initialization storage", source))?; let _receipt = initialize_store(&mut storage) .map_err(|source| verification("initialize production crash store", source))?; - let lock = storage.into_writer_lock().map_err(|source| { - DurabilityCrashMatrixError::io("retain initialized writer lock", source) - })?; + storage + .into_writer_lock() + .map_err(|source| DurabilityCrashMatrixError::io("retain initialized writer lock", source)) +} + +pub(super) fn publisher( + store_root: &Path, +) -> Result { + let lock = initialized_lock(store_root)?; FilesystemCatalogPublisher::open_unchecked_for_repository_tasks(lock, restart_policy()?) .map_err(|source| DurabilityCrashMatrixError::io("open crash catalog publisher", source)) } diff --git a/xtask/src/durability_crash_matrix/production_protocol/retention.rs b/xtask/src/durability_crash_matrix/production_protocol/retention.rs new file mode 100644 index 0000000..486f54b --- /dev/null +++ b/xtask/src/durability_crash_matrix/production_protocol/retention.rs @@ -0,0 +1,114 @@ +//! This module owns execution of the production retention publication protocol. + +use std::fs; +use std::path::Path; + +use keep::{ + AdmittedCatalog, AdmittedRetentionRoot, AdmittedSegment, ChecksummedCatalog, + ChecksummedPublicationHead, FilesystemRetentionPublicationAuthority, + FilesystemStoreMigrationAuthority, FilesystemVersionTwoAdmission, + RetentionGenerationExpectation, RetentionPublicationPreparation, execute_retention_publication, + execute_store_migration, preflight_retention_transition, prepare_retention_publication, +}; + +use super::control::CrashControl; +use super::fixture::{BUNDLE_CATALOG_NAME, BUNDLE_SEGMENT_NAME, GoldenFixture}; +use super::initialization; +use super::retention_storage::CrashRetentionStorage; +use super::{DurabilityCrashMatrixError, verification}; + +/// Migrates a fresh bundle store and publishes retention generation one, +/// dying at the selected coordinate. +pub(super) fn run( + store_root: &Path, + control: &mut CrashControl, +) -> Result<(), DurabilityCrashMatrixError> { + let authority = migrated_authority(store_root)?; + let root = GoldenFixture::retention_root()?; + let preparation = preparation(root.bytes())?; + let mut storage = CrashRetentionStorage::new(authority, control, store_root); + execute_retention_publication(&mut storage, &preparation) + .map(|_receipt| ()) + .map_err(|source| verification("execute production retention publication", source)) +} + +/// Initializes, populates, and migrates the bundle store, then reopens it as +/// version two and returns retention authority over it. +fn migrated_authority( + store_root: &Path, +) -> Result { + let lock = initialization::initialized_lock(store_root)?; + write_bundle(store_root)?; + let mut migration = FilesystemStoreMigrationAuthority::open_unchecked_for_repository_tasks( + lock, + initialization::segment_policy(), + ) + .map_err(|source| verification("open crash migration authority", source))?; + let intent = migration + .observe_intent() + .map_err(|source| verification("observe crash migration intent", source))?; + let _receipt = execute_store_migration(&mut migration, &intent) + .map_err(|source| verification("execute crash store migration", source))?; + drop(migration); + reopened_authority(store_root) +} + +/// Reopens the migrated store and returns retention authority over it. +pub(in crate::durability_crash_matrix) fn reopened_authority( + store_root: &Path, +) -> Result { + let admission = + FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(store_root) + .map_err(|source| verification("reopen crash store as version two", source))?; + FilesystemRetentionPublicationAuthority::open(admission) + .map_err(|source| verification("open crash retention authority", source)) +} + +fn write_bundle(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { + let segment = GoldenFixture::bundle_segment()?; + let catalog = GoldenFixture::bundle_catalog()?; + let head = GoldenFixture::bundle_head()?; + for (relative, bytes) in [ + (format!("segments/{BUNDLE_SEGMENT_NAME}"), segment.bytes()), + (format!("catalogs/{BUNDLE_CATALOG_NAME}"), catalog.bytes()), + ("HEAD".to_owned(), head.bytes()), + ] { + fs::write(store_root.join(&relative), bytes) + .map_err(|source| DurabilityCrashMatrixError::io("write bundle corpus", source))?; + } + Ok(()) +} + +/// Prepares the frozen generation-one root as an initial publication against +/// the bundle catalog snapshot. +pub(in crate::durability_crash_matrix) fn preparation( + root_bytes: &[u8], +) -> Result, DurabilityCrashMatrixError> { + let segment_fixture = GoldenFixture::bundle_segment()?; + let catalog_fixture = GoldenFixture::bundle_catalog()?; + let head_fixture = GoldenFixture::bundle_head()?; + let candidate = AdmittedRetentionRoot::decode(root_bytes) + .map_err(|source| verification("decode crash retention root", source))?; + let segment = + AdmittedSegment::decode(segment_fixture.bytes(), initialization::segment_policy()) + .map_err(|source| verification("admit bundle segment", source))?; + let segments = [segment]; + let catalog: AdmittedCatalog<'_, '_> = ChecksummedCatalog::decode(catalog_fixture.bytes()) + .map_err(|source| verification("decode bundle catalog", source))? + .admit(&segments) + .map_err(|source| verification("admit bundle catalog", source))?; + let head = ChecksummedPublicationHead::decode(head_fixture.bytes()) + .map_err(|source| verification("decode bundle head", source))?; + let snapshot = head + .admit(catalog) + .map_err(|source| verification("admit bundle snapshot", source))?; + let preflight = preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + &snapshot, + ) + .map_err(|source| verification("preflight crash retention transition", source))?; + prepare_retention_publication(preflight, None) + .map_err(|source| verification("prepare crash retention publication", source)) +} diff --git a/xtask/src/durability_crash_matrix/production_protocol/retention_storage.rs b/xtask/src/durability_crash_matrix/production_protocol/retention_storage.rs new file mode 100644 index 0000000..3d8bfa2 --- /dev/null +++ b/xtask/src/durability_crash_matrix/production_protocol/retention_storage.rs @@ -0,0 +1,229 @@ +//! This module owns crash injection around production retention publication. + +use std::fs::OpenOptions; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; + +use keep::{ + AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, + FilesystemRetentionPublicationAuthority, RetentionNamespaceAdmission, + RetentionPublicationPreparation, RetentionPublicationStorage, RetentionTransitionDisposition, +}; +use xtask::{DurabilityCrashPoint, DurabilityCrashPosition}; + +use super::control::{CrashControl, DuringTiming}; + +/// Bytes an interrupted stage write leaves behind: inside every record's +/// fixed framing, so restart classifies the stage as truncated. +const STAGE_INTERRUPTION: usize = 100; + +pub(super) struct CrashRetentionStorage<'control> { + inner: FilesystemRetentionPublicationAuthority, + control: &'control mut CrashControl, + retention: PathBuf, +} + +impl<'control> CrashRetentionStorage<'control> { + pub(super) fn new( + inner: FilesystemRetentionPublicationAuthority, + control: &'control mut CrashControl, + store_root: &Path, + ) -> Self { + Self { + inner, + control, + retention: store_root.join("retention"), + } + } + + fn execute( + &mut self, + point: DurabilityCrashPoint, + during: DuringTiming, + operation: impl FnOnce(&mut FilesystemRetentionPublicationAuthority) -> io::Result, + ) -> io::Result { + self.control.before(point, during)?; + let result = operation(&mut self.inner)?; + self.control.after(point, during)?; + Ok(result) + } + + fn execute_write( + &mut self, + point: DurabilityCrashPoint, + stage: &str, + bytes: &[u8], + complete: impl FnOnce(&mut FilesystemRetentionPublicationAuthority) -> io::Result<()>, + ) -> io::Result<()> { + match self.control.position(point) { + None => complete(&mut self.inner), + Some(DurabilityCrashPosition::Before) => self.control.await_process_death(), + Some(DurabilityCrashPosition::During) => { + let partial = bytes.get(..STAGE_INTERRUPTION).ok_or_else(|| { + io::Error::other("retention record shorter than the interruption prefix") + })?; + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(self.retention.join(stage))?; + file.write_all(partial)?; + self.control.await_process_death() + } + Some(DurabilityCrashPosition::After) => { + complete(&mut self.inner)?; + self.control.await_process_death() + } + } + } +} + +impl RetentionPublicationStorage for CrashRetentionStorage<'_> { + fn verify_current( + &mut self, + preparation: &RetentionPublicationPreparation<'_>, + ) -> io::Result { + self.inner.verify_current(preparation) + } + + fn write_root_stage(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { + self.execute_write( + DurabilityCrashPoint::WriteRootStage, + "root.next", + root.encoded(), + |inner| inner.write_root_stage(root), + ) + } + + fn synchronize_root_stage(&mut self) -> io::Result<()> { + self.execute( + DurabilityCrashPoint::SynchronizeRootStage, + DuringTiming::Before, + FilesystemRetentionPublicationAuthority::synchronize_root_stage, + ) + } + + fn admit_root_namespace( + &mut self, + root: &AdmittedRetentionRoot<'_>, + ) -> io::Result { + self.execute( + DurabilityCrashPoint::AdmitRootNamespace, + DuringTiming::After, + |inner| inner.admit_root_namespace(root), + ) + } + + fn synchronize_roots_after_namespace(&mut self) -> io::Result<()> { + self.execute( + DurabilityCrashPoint::SynchronizeRootsAfterNamespace, + DuringTiming::Before, + FilesystemRetentionPublicationAuthority::synchronize_roots_after_namespace, + ) + } + + fn link_root(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { + self.execute( + DurabilityCrashPoint::LinkRoot, + DuringTiming::After, + |inner| inner.link_root(root), + ) + } + + fn synchronize_root_namespace(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { + self.execute( + DurabilityCrashPoint::SynchronizeRootNamespace, + DuringTiming::Before, + |inner| inner.synchronize_root_namespace(root), + ) + } + + fn write_manifest_stage(&mut self, manifest: &CanonicalRetentionManifest) -> io::Result<()> { + self.execute_write( + DurabilityCrashPoint::WriteManifestStage, + "manifest.next", + manifest.encoded(), + |inner| inner.write_manifest_stage(manifest), + ) + } + + fn synchronize_manifest_stage(&mut self) -> io::Result<()> { + self.execute( + DurabilityCrashPoint::SynchronizeManifestStage, + DuringTiming::Before, + FilesystemRetentionPublicationAuthority::synchronize_manifest_stage, + ) + } + + fn link_manifest(&mut self, manifest: &CanonicalRetentionManifest) -> io::Result<()> { + self.execute( + DurabilityCrashPoint::LinkManifest, + DuringTiming::After, + |inner| inner.link_manifest(manifest), + ) + } + + fn synchronize_manifest_pool(&mut self) -> io::Result<()> { + self.execute( + DurabilityCrashPoint::SynchronizeManifestPool, + DuringTiming::Before, + FilesystemRetentionPublicationAuthority::synchronize_manifest_pool, + ) + } + + fn write_head_stage(&mut self, head: &CanonicalRetentionHead) -> io::Result<()> { + self.execute_write( + DurabilityCrashPoint::WriteHeadStage, + "head.next", + head.encoded(), + |inner| inner.write_head_stage(head), + ) + } + + fn synchronize_head_stage(&mut self) -> io::Result<()> { + self.execute( + DurabilityCrashPoint::SynchronizeHeadStage, + DuringTiming::Before, + FilesystemRetentionPublicationAuthority::synchronize_head_stage, + ) + } + + fn replace_head(&mut self) -> io::Result<()> { + self.execute( + DurabilityCrashPoint::ReplaceRetentionHead, + DuringTiming::After, + FilesystemRetentionPublicationAuthority::replace_head, + ) + } + + fn synchronize_retention_namespace(&mut self) -> io::Result<()> { + self.execute( + DurabilityCrashPoint::SynchronizeRetentionNamespace, + DuringTiming::Before, + FilesystemRetentionPublicationAuthority::synchronize_retention_namespace, + ) + } + + fn remove_root_stage(&mut self) -> io::Result<()> { + self.execute( + DurabilityCrashPoint::RemoveRootStage, + DuringTiming::After, + FilesystemRetentionPublicationAuthority::remove_root_stage, + ) + } + + fn remove_manifest_stage(&mut self) -> io::Result<()> { + self.execute( + DurabilityCrashPoint::RemoveManifestStage, + DuringTiming::After, + FilesystemRetentionPublicationAuthority::remove_manifest_stage, + ) + } + + fn synchronize_cleanup(&mut self) -> io::Result<()> { + self.execute( + DurabilityCrashPoint::SynchronizeRetentionCleanup, + DuringTiming::Before, + FilesystemRetentionPublicationAuthority::synchronize_cleanup, + ) + } +} diff --git a/xtask/src/durability_crash_matrix/restart.rs b/xtask/src/durability_crash_matrix/restart.rs index 423efc8..aedabe8 100644 --- a/xtask/src/durability_crash_matrix/restart.rs +++ b/xtask/src/durability_crash_matrix/restart.rs @@ -1,6 +1,7 @@ //! This module owns independent post-process-death store verification. mod expectation; +mod retention; mod semantic; use std::collections::BTreeSet; @@ -17,6 +18,9 @@ pub(super) fn verify( store_root: &Path, case: DurabilityCrashCase, ) -> Result<(), DurabilityCrashMatrixError> { + if case.point().sequence() == xtask::DurabilityCrashSequence::Retention { + return retention::verify(store_root, case); + } let expected = ExpectedStoreState::for_case(case)?; let observed_paths = inventory(store_root)?; if observed_paths != expected.paths() { diff --git a/xtask/src/durability_crash_matrix/restart/expectation.rs b/xtask/src/durability_crash_matrix/restart/expectation.rs index 1b6be39..0d000ca 100644 --- a/xtask/src/durability_crash_matrix/restart/expectation.rs +++ b/xtask/src/durability_crash_matrix/restart/expectation.rs @@ -64,6 +64,11 @@ impl ExpectedStoreState { DurabilityCrashSequence::Head => sequence::head(case), DurabilityCrashSequence::RecoveryDiscard => sequence::recovery(case), DurabilityCrashSequence::Initialization => sequence::initialization(case), + DurabilityCrashSequence::Retention => { + Err(DurabilityCrashMatrixError::PointSequenceMismatch { + point: case.point(), + }) + } } } diff --git a/xtask/src/durability_crash_matrix/restart/retention.rs b/xtask/src/durability_crash_matrix/restart/retention.rs new file mode 100644 index 0000000..e210a18 --- /dev/null +++ b/xtask/src/durability_crash_matrix/restart/retention.rs @@ -0,0 +1,162 @@ +//! This module owns post-process-death verification of retention publication. +//! +//! After the child dies at its coordinate, restart reopens the migrated store +//! through the same admission a production caller would use, runs retention +//! recovery, and requires the documented steps and outcome for that exact +//! prefix; then it requires the forward retry to report the outcome recovery +//! predicts. + +use std::io; +use std::path::Path; + +use keep::{ + RetentionCurrentStateRefusal, RetentionPublicationError, RetentionPublicationOutcome, + RetentionRecoveryOutcome as Outcome, RetentionRecoveryStep as Step, + execute_retention_publication, +}; +use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; + +use super::super::DurabilityCrashMatrixError; +use super::super::production_protocol::fixture::GoldenFixture; +use super::super::production_protocol::retention::{preparation, reopened_authority}; +use super::super::production_protocol::verification; + +const PROTECTED_ROOT: Outcome = Outcome::Protected { + root_stage: true, + manifest_stage: false, +}; +const PROTECTED_BOTH: Outcome = Outcome::Protected { + root_stage: true, + manifest_stage: true, +}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Retry { + Published, + AlreadyCommitted, + Refused, +} + +pub(super) fn verify( + store_root: &Path, + case: DurabilityCrashCase, +) -> Result<(), DurabilityCrashMatrixError> { + let (steps, outcome, retry) = expected(case); + let mut authority = reopened_authority(store_root)?; + let receipt = authority + .recover() + .map_err(|source| verification("recover crash retention stages", source))?; + if receipt.executed() != steps.as_slice() || receipt.outcome() != outcome { + return Err(mismatch(format!( + "{} {:?}: expected {steps:?} -> {outcome:?}, recovered {:?} -> {:?}", + case.point().identifier(), + case.position(), + receipt.executed(), + receipt.outcome() + ))); + } + let root = GoldenFixture::retention_root()?; + let preparation = preparation(root.bytes())?; + match ( + retry, + execute_retention_publication(&mut authority, &preparation), + ) { + (Retry::Published, Ok(receipt)) + if receipt.outcome() == RetentionPublicationOutcome::Published => {} + (Retry::AlreadyCommitted, Ok(receipt)) + if receipt.outcome() == RetentionPublicationOutcome::AlreadyCommitted => {} + (Retry::Refused, Err(RetentionPublicationError::CurrentVerification { source })) + if source + .get_ref() + .and_then(|refusal| refusal.downcast_ref::()) + .is_some_and(|refusal| { + matches!(refusal, RetentionCurrentStateRefusal::RetainedStage) + }) => {} + (retry, result) => { + return Err(mismatch(format!( + "{} {:?}: expected forward retry {retry:?}, got {result:?}", + case.point().identifier(), + case.position() + ))); + } + } + Ok(()) +} + +fn mismatch(message: String) -> DurabilityCrashMatrixError { + verification("verify crash retention recovery", io::Error::other(message)) +} + +/// The number of completed publication phases and any truncated stage the +/// coordinate leaves behind. +fn prefix(case: DurabilityCrashCase) -> (usize, Option) { + let index = DurabilityCrashPoint::ALL + .iter() + .position(|point| *point == case.point()) + .and_then(|index| index.checked_sub(35)) + .unwrap_or(0); + // Phase 1 is current-state verification; point n is phase n + 2. + let phase = index.saturating_add(2); + let write = match case.point() { + DurabilityCrashPoint::WriteRootStage => Some(Step::DiscardRootStage), + DurabilityCrashPoint::WriteManifestStage => Some(Step::DiscardManifestStage), + DurabilityCrashPoint::WriteHeadStage => Some(Step::DiscardHeadStage), + _ => None, + }; + match case.position() { + DurabilityCrashPosition::After => (phase, None), + DurabilityCrashPosition::During if write.is_some() => (phase.saturating_sub(1), write), + DurabilityCrashPosition::During if atomic(case.point()) => (phase, None), + DurabilityCrashPosition::Before | DurabilityCrashPosition::During => { + (phase.saturating_sub(1), None) + } + } +} + +const fn atomic(point: DurabilityCrashPoint) -> bool { + matches!( + point, + DurabilityCrashPoint::AdmitRootNamespace + | DurabilityCrashPoint::LinkRoot + | DurabilityCrashPoint::LinkManifest + | DurabilityCrashPoint::ReplaceRetentionHead + | DurabilityCrashPoint::RemoveRootStage + | DurabilityCrashPoint::RemoveManifestStage + ) +} + +/// The documented recovery for the prefix a coordinate leaves behind. +fn expected(case: DurabilityCrashCase) -> (Vec, Outcome, Retry) { + let (count, truncated) = prefix(case); + let (mut steps, outcome, retry) = match count { + 0 | 1 => (vec![], Outcome::Clean, Retry::Published), + 2..=5 => (vec![Step::LinkRoot], PROTECTED_ROOT, Retry::Refused), + 6 | 7 => (vec![], PROTECTED_ROOT, Retry::Refused), + 8 | 9 => (vec![Step::LinkManifest], PROTECTED_BOTH, Retry::Refused), + 10 | 11 => (vec![], PROTECTED_BOTH, Retry::Refused), + 12 | 13 => ( + vec![ + Step::FinalizeHead, + Step::RemoveRootStage, + Step::RemoveManifestStage, + ], + Outcome::Committed, + Retry::AlreadyCommitted, + ), + 14 | 15 => ( + vec![Step::RemoveRootStage, Step::RemoveManifestStage], + Outcome::Committed, + Retry::AlreadyCommitted, + ), + 16 => ( + vec![Step::RemoveManifestStage], + Outcome::Committed, + Retry::AlreadyCommitted, + ), + _ => (vec![], Outcome::Clean, Retry::AlreadyCommitted), + }; + if let Some(discard) = truncated { + steps.insert(0, discard); + } + (steps, outcome, retry) +} diff --git a/xtask/src/durability_crash_point.rs b/xtask/src/durability_crash_point.rs index 2ed8d6a..bbb01a2 100644 --- a/xtask/src/durability_crash_point.rs +++ b/xtask/src/durability_crash_point.rs @@ -13,6 +13,8 @@ pub enum DurabilityCrashSequence { RecoveryDiscard, /// Writer-locked store initialization. Initialization, + /// Version-two retention publication, `KEEP-CRASH-036` through `052`. + Retention, } /// One stable process-death boundary in the durable segment-store protocol. @@ -88,11 +90,45 @@ pub enum DurabilityCrashPoint { CreateCatalogPoolDirectory, /// Synchronize the store root after initialization. SynchronizeRootAfterInitialization, + /// Retention root stage write. + WriteRootStage, + /// Retention root stage synchronization. + SynchronizeRootStage, + /// New namespace-directory creation or exact admission. + AdmitRootNamespace, + /// Namespace-pool synchronization after creation. + SynchronizeRootsAfterNamespace, + /// Immutable root link. + LinkRoot, + /// Root namespace-directory synchronization. + SynchronizeRootNamespace, + /// Retention manifest stage write. + WriteManifestStage, + /// Retention manifest stage synchronization. + SynchronizeManifestStage, + /// Immutable manifest link. + LinkManifest, + /// Manifest pool synchronization. + SynchronizeManifestPool, + /// Retention-head stage write. + WriteHeadStage, + /// Retention-head stage synchronization. + SynchronizeHeadStage, + /// Retention-head atomic replacement. + ReplaceRetentionHead, + /// Committed retention namespace synchronization. + SynchronizeRetentionNamespace, + /// Retained root-stage removal. + RemoveRootStage, + /// Retained manifest-stage removal. + RemoveManifestStage, + /// Retention cleanup synchronization. + SynchronizeRetentionCleanup, } impl DurabilityCrashPoint { /// Every crash boundary in stable protocol order. - pub const ALL: [Self; 35] = [ + pub const ALL: [Self; 52] = [ Self::CreateSegmentStage, Self::WriteSegmentHeader, Self::AppendSegmentRecord, @@ -128,6 +164,23 @@ impl DurabilityCrashPoint { Self::CreateSegmentPoolDirectory, Self::CreateCatalogPoolDirectory, Self::SynchronizeRootAfterInitialization, + Self::WriteRootStage, + Self::SynchronizeRootStage, + Self::AdmitRootNamespace, + Self::SynchronizeRootsAfterNamespace, + Self::LinkRoot, + Self::SynchronizeRootNamespace, + Self::WriteManifestStage, + Self::SynchronizeManifestStage, + Self::LinkManifest, + Self::SynchronizeManifestPool, + Self::WriteHeadStage, + Self::SynchronizeHeadStage, + Self::ReplaceRetentionHead, + Self::SynchronizeRetentionNamespace, + Self::RemoveRootStage, + Self::RemoveManifestStage, + Self::SynchronizeRetentionCleanup, ]; /// Parses one exact stable crash identifier. @@ -177,6 +230,23 @@ impl DurabilityCrashPoint { | Self::CreateSegmentPoolDirectory | Self::CreateCatalogPoolDirectory | Self::SynchronizeRootAfterInitialization => DurabilityCrashSequence::Initialization, + Self::WriteRootStage + | Self::SynchronizeRootStage + | Self::AdmitRootNamespace + | Self::SynchronizeRootsAfterNamespace + | Self::LinkRoot + | Self::SynchronizeRootNamespace + | Self::WriteManifestStage + | Self::SynchronizeManifestStage + | Self::LinkManifest + | Self::SynchronizeManifestPool + | Self::WriteHeadStage + | Self::SynchronizeHeadStage + | Self::ReplaceRetentionHead + | Self::SynchronizeRetentionNamespace + | Self::RemoveRootStage + | Self::RemoveManifestStage + | Self::SynchronizeRetentionCleanup => DurabilityCrashSequence::Retention, } } diff --git a/xtask/src/durability_crash_point_identity.rs b/xtask/src/durability_crash_point_identity.rs index 173afa7..4ed0bc5 100644 --- a/xtask/src/durability_crash_point_identity.rs +++ b/xtask/src/durability_crash_point_identity.rs @@ -42,6 +42,23 @@ impl DurabilityCrashPoint { Self::CreateSegmentPoolDirectory => "KEEP-CRASH-033", Self::CreateCatalogPoolDirectory => "KEEP-CRASH-034", Self::SynchronizeRootAfterInitialization => "KEEP-CRASH-035", + Self::WriteRootStage => "KEEP-CRASH-036", + Self::SynchronizeRootStage => "KEEP-CRASH-037", + Self::AdmitRootNamespace => "KEEP-CRASH-038", + Self::SynchronizeRootsAfterNamespace => "KEEP-CRASH-039", + Self::LinkRoot => "KEEP-CRASH-040", + Self::SynchronizeRootNamespace => "KEEP-CRASH-041", + Self::WriteManifestStage => "KEEP-CRASH-042", + Self::SynchronizeManifestStage => "KEEP-CRASH-043", + Self::LinkManifest => "KEEP-CRASH-044", + Self::SynchronizeManifestPool => "KEEP-CRASH-045", + Self::WriteHeadStage => "KEEP-CRASH-046", + Self::SynchronizeHeadStage => "KEEP-CRASH-047", + Self::ReplaceRetentionHead => "KEEP-CRASH-048", + Self::SynchronizeRetentionNamespace => "KEEP-CRASH-049", + Self::RemoveRootStage => "KEEP-CRASH-050", + Self::RemoveManifestStage => "KEEP-CRASH-051", + Self::SynchronizeRetentionCleanup => "KEEP-CRASH-052", } } } diff --git a/xtask/tests/durability_crash_point_contract.rs b/xtask/tests/durability_crash_point_contract.rs index 1945b01..61b3543 100644 --- a/xtask/tests/durability_crash_point_contract.rs +++ b/xtask/tests/durability_crash_point_contract.rs @@ -4,7 +4,7 @@ use xtask::{DurabilityCrashPoint, DurabilityCrashSequence}; -use DurabilityCrashSequence::{Catalog, Head, Initialization, RecoveryDiscard, Segment}; +use DurabilityCrashSequence::{Catalog, Head, Initialization, RecoveryDiscard, Retention, Segment}; const EXPECTED: &[(DurabilityCrashPoint, &str, DurabilityCrashSequence)] = &[ ( @@ -162,6 +162,87 @@ const EXPECTED: &[(DurabilityCrashPoint, &str, DurabilityCrashSequence)] = &[ "KEEP-CRASH-035", Initialization, ), + ( + DurabilityCrashPoint::WriteRootStage, + "KEEP-CRASH-036", + Retention, + ), + ( + DurabilityCrashPoint::SynchronizeRootStage, + "KEEP-CRASH-037", + Retention, + ), + ( + DurabilityCrashPoint::AdmitRootNamespace, + "KEEP-CRASH-038", + Retention, + ), + ( + DurabilityCrashPoint::SynchronizeRootsAfterNamespace, + "KEEP-CRASH-039", + Retention, + ), + (DurabilityCrashPoint::LinkRoot, "KEEP-CRASH-040", Retention), + ( + DurabilityCrashPoint::SynchronizeRootNamespace, + "KEEP-CRASH-041", + Retention, + ), + ( + DurabilityCrashPoint::WriteManifestStage, + "KEEP-CRASH-042", + Retention, + ), + ( + DurabilityCrashPoint::SynchronizeManifestStage, + "KEEP-CRASH-043", + Retention, + ), + ( + DurabilityCrashPoint::LinkManifest, + "KEEP-CRASH-044", + Retention, + ), + ( + DurabilityCrashPoint::SynchronizeManifestPool, + "KEEP-CRASH-045", + Retention, + ), + ( + DurabilityCrashPoint::WriteHeadStage, + "KEEP-CRASH-046", + Retention, + ), + ( + DurabilityCrashPoint::SynchronizeHeadStage, + "KEEP-CRASH-047", + Retention, + ), + ( + DurabilityCrashPoint::ReplaceRetentionHead, + "KEEP-CRASH-048", + Retention, + ), + ( + DurabilityCrashPoint::SynchronizeRetentionNamespace, + "KEEP-CRASH-049", + Retention, + ), + ( + DurabilityCrashPoint::RemoveRootStage, + "KEEP-CRASH-050", + Retention, + ), + ( + DurabilityCrashPoint::RemoveManifestStage, + "KEEP-CRASH-051", + Retention, + ), + ( + DurabilityCrashPoint::SynchronizeRetentionCleanup, + "KEEP-CRASH-052", + Retention, + ), ]; #[test] From 4aad28d13022a7c408e0be97e50fb2cdc2b8d736 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 9 Sep 2026 14:35:30 -0700 Subject: [PATCH 7/8] Add: fenced, double-collected reader views of a version-two store Readers had no way to observe a version-two store that could not straddle a publication: nothing held the reader fence the recovery page specifies, and nothing bound the catalog head and the retention head to one instant. ReaderFence acquires a shared kernel lock on reader.lock, verified as a regular zero-length file reached without following links and re-verified after locking, and holds it for the snapshot's lifetime; collection will take the same lock exclusively, so no published root, manifest, or segment can be deleted under a live view. collect_retention_view is storage-independent: it reads both head coordinates, loads the view, reads them again, and accepts only agreement, retrying within a ReaderAttemptLimit and refusing an exhausted limit or an absent catalog. FilesystemRetentionSnapshot admits the root as version two, acquires the fence, collects the catalog snapshot, the retention head, and its manifest through that loop, and verifies each selected root against the manifest on demand while the fence is held. Four scripted-source laws pin the loop (first-attempt acceptance, retry after a publication between the reads, exhaustion, absent catalog). Five filesystem laws pin the fence and the view: an unpublished store binds the catalog and no head; a published generation is read and its root verified byte for byte; a substituted root refuses; two readers share the fence while an exclusive lock waits; a replaced reader.lock refuses. KEEP-RETENTION-008 is Implemented in the ledger; the README's fence gap is closed. Refs #19 --- CHANGELOG.md | 7 + README.md | 6 +- docs/formats/segment-store-v2/README.md | 4 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/retention.rs | 16 ++ .../filesystem_retention_snapshot.rs | 224 ++++++++++++++++++ .../filesystem_retention_snapshot_error.rs | 63 +++++ .../filesystem_retention_snapshot_tests.rs | 124 ++++++++++ .../retention/reader_attempt_limit.rs | 25 ++ src/adapters/retention/reader_fence.rs | 60 +++++ .../retention/retention_view_collector.rs | 113 +++++++++ .../retention_view_collector_tests.rs | 104 ++++++++ src/lib.rs | 33 +-- 13 files changed, 761 insertions(+), 20 deletions(-) create mode 100644 src/adapters/retention/filesystem_retention_snapshot.rs create mode 100644 src/adapters/retention/filesystem_retention_snapshot_error.rs create mode 100644 src/adapters/retention/filesystem_retention_snapshot_tests.rs create mode 100644 src/adapters/retention/reader_attempt_limit.rs create mode 100644 src/adapters/retention/reader_fence.rs create mode 100644 src/adapters/retention/retention_view_collector.rs create mode 100644 src/adapters/retention/retention_view_collector_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 93f740f..da0a2f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ after its public API and format compatibility policies are established. ### Added +- `FilesystemRetentionSnapshot` is the version-two reader view: it admits the + root as version two, acquires a shared `ReaderFence` on `reader.lock`, + double-collects the catalog and retention heads around loading through + `collect_retention_view` (bounded by `ReaderAttemptLimit`, refusing an + exhausted limit or an absent catalog), binds the catalog snapshot, the + retention head, and its manifest, and verifies each selected root against + the manifest on demand while the fence is held. - Storage-independent retention recovery planning: `assess_root_stage`, `assess_manifest_stage`, and `assess_head_stage` classify each fixed stage as absent, complete, truncated, or corrupt through the decoders' own diff --git a/README.md b/README.md index cedc6fa..3f5dc4f 100644 --- a/README.md +++ b/README.md @@ -76,14 +76,14 @@ retry reports already committed. The one state that waits for a human is a complete orphan, a crash between the root link and the head finalization, which stays recovery-protected until explicit disposition lands with garbage collection (#21). The crash matrix proves that recovery by killing real -writer processes at all 51 retention coordinates. Readers have no fence yet -(#19). A version-1 store stays admitted until its owner migrates it. +writer processes at all 51 retention coordinates. Readers hold a shared +fence and double-collect both heads, so a view never straddles a +publication. A version-1 store stays admitted until its owner migrates it. | Gap | Tracked | | --- | --- | | Restart recovery for retention publication and migration | [#19](https://github.com/flyingrobots/keep/issues/19) | | Restart-stable root identity coordinate in the migration intent | [#97](https://github.com/flyingrobots/keep/issues/97) | -| Reader fence binding one consistent catalog + retention snapshot | [#19](https://github.com/flyingrobots/keep/issues/19) | | Precise verification reports at explicit depths | [#20](https://github.com/flyingrobots/keep/issues/20) | | Garbage collection and identity-preserving compaction | [#21](https://github.com/flyingrobots/keep/issues/21) | | Bounded production ingestion through the durable store | [#82](https://github.com/flyingrobots/keep/issues/82) | diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index e3a8274..65b9e47 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -97,8 +97,10 @@ violation before mutation, each as a typed `RetentionCurrentStateRefusal`. Retention publication recovery is implemented and proven both in-process for every crash prefix and by the crash matrix, which kills a real writer before, during, and after `KEEP-CRASH-036` through `052`. +Readers bind one consistent catalog, retention head, and manifest view under a +shared `ReaderFence` and verify selected roots on demand. Not implemented: partial-prefix migration recovery and `KEEP-CRASH-053..073`, -the reader fence, model-based transition evidence, and garbage collection. Issue #19 owns the first four and issue #21 the last; +model-based transition evidence, and garbage collection. Issue #19 owns the first four and issue #21 the last; issue #97 owns the restart-stable root identity coordinate. A version-1 store remains admitted until its owner migrates it, and the [requirements ledger](requirements.md) is the authority on which requirements diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 6d1d318..2336887 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -16,7 +16,7 @@ case is not evidence. | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; production 17-phase forward filesystem execution, exclusive staging, byte-equal inode-substitution refusal, and retained-stage recovery refusal in `filesystem_retention_storage_tests`; orphan namespace directories count against the 4,096 ceiling and refuse a new namespace before any stage is written in `filesystem_retention_capacity_tests`; crash injection remains | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | recovery-required refusals before any mutation in `filesystem_retention_expectation_tests`: an absent head over populated pools, a non-initial head prepared against an absent head, an orphan directory for a namespace expected absent, and an absent directory for a namespace expected current; replaced protocol directories, an absent or changed head-selected catalog, an over-full census, zero-generation pool names, and a stage retained by a failed write refuse in `filesystem_retention_*_tests`; storage-independent classification of every fixed-stage crash prefix (discard, link and protect, finalize, clean up, or typed refusal) in `recovery_planner_tests`; every publication prefix 0 through 18, each mid-write truncation, and successor prefixes recover in-process to the documented state, idempotently, with the forward retry reporting the predicted outcome, in `filesystem_retention_recovery_prefix_tests`; `cargo xtask durability-crash-matrix` kills a real writer before, during, and after `KEEP-CRASH-036` through `052` and requires restart recovery to reach the documented state and the forward retry to report the predicted outcome | Implemented | -| `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | +| `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | `ReaderFence` holds a shared kernel lock on a verified zero-length `reader.lock`; `collect_retention_view` accepts a view only when both head coordinates agree before and after loading and refuses an exhausted attempt limit (`retention_view_collector_tests`); `FilesystemRetentionSnapshot` binds the catalog snapshot, retention head, and manifest under the fence and verifies each selected root on demand while the fence is held, refusing a substituted root and a replaced fence, and two readers share the fence while an exclusive lock waits (`filesystem_retention_snapshot_tests`) | Implemented | | `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests`; committed retry reopens the head-selected manifest entry and root pool bytes, refusing absent, changed, or corrupt evidence in `filesystem_retention_current_tests`; every refusal is a typed `RetentionCurrentStateRefusal` source, with superseded, committed-root-absent, committed-root-changed, and head-absent-with-artifacts pinned by downcast | Implemented | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | model-based and source-architecture tests | Planned in #19 | diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 3dabcda..84b6c29 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -44,6 +44,10 @@ mod filesystem_retention_recovery_prefix_tests; #[cfg(test)] mod filesystem_retention_recovery_tests; mod filesystem_retention_refusal; +mod filesystem_retention_snapshot; +mod filesystem_retention_snapshot_error; +#[cfg(test)] +mod filesystem_retention_snapshot_tests; mod filesystem_retention_stage; mod filesystem_retention_storage; #[cfg(test)] @@ -98,6 +102,8 @@ mod transition_preflight_error; mod transition_readiness; mod verified_closure; +mod reader_attempt_limit; +mod reader_fence; mod recovery_evidence; mod recovery_execution; #[cfg(test)] @@ -109,6 +115,9 @@ mod recovery_planner_tests; mod recovery_refusal; mod recovery_stage_assessment; mod recovery_storage; +mod retention_view_collector; +#[cfg(test)] +mod retention_view_collector_tests; pub use admitted_manifest::AdmittedRetentionManifest; pub use admitted_root::AdmittedRetentionRoot; pub use canonical_head::CanonicalRetentionHead; @@ -124,6 +133,8 @@ pub use filesystem_retention_authority_error::{ pub use filesystem_retention_current::ObservedRetentionState; pub use filesystem_retention_recovery_error::FilesystemRetentionRecoveryError; pub use filesystem_retention_refusal::RetentionCurrentStateRefusal; +pub use filesystem_retention_snapshot::FilesystemRetentionSnapshot; +pub use filesystem_retention_snapshot_error::FilesystemRetentionSnapshotError; pub use head_decode_error::RetentionHeadDecodeError; pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; @@ -137,6 +148,8 @@ pub use publication_preparation::prepare_retention_publication; pub use publication_preparation_error::RetentionPublicationPreparationError; pub use publication_receipt::RetentionPublicationReceipt; pub use publication_storage::RetentionPublicationStorage; +pub use reader_attempt_limit::ReaderAttemptLimit; +pub use reader_fence::ReaderFence; pub use recovery_evidence::{ RetentionPoolEntryObservation, RetentionPoolObservations, RetentionRecoveryEvidence, RetentionStageAssessments, @@ -152,6 +165,9 @@ pub use recovery_stage_assessment::{ RetentionStageAssessment, assess_head_stage, assess_manifest_stage, assess_root_stage, }; pub use recovery_storage::RetentionRecoveryStorage; +pub use retention_view_collector::{ + RetentionViewCoordinates, RetentionViewError, RetentionViewSource, collect_retention_view, +}; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; pub use transition_disposition::RetentionTransitionDisposition; diff --git a/src/adapters/retention/filesystem_retention_snapshot.rs b/src/adapters/retention/filesystem_retention_snapshot.rs new file mode 100644 index 0000000..131947e --- /dev/null +++ b/src/adapters/retention/filesystem_retention_snapshot.rs @@ -0,0 +1,224 @@ +//! This module owns one fenced, double-collected reader view of a version-two store. + +use std::io; +use std::path::{Path, PathBuf}; + +use cap_fs_ext::DirExt; +use cap_std::fs::Dir; + +use super::filesystem_retention_current::{self, ObservedRetentionState}; +use super::filesystem_retention_pool_name as pool_name; +use super::{ + AdmittedRetentionRoot, FilesystemRetentionSnapshotError as Error, ReaderAttemptLimit, + ReaderFence, RetentionViewCoordinates, RetentionViewSource, collect_retention_view, + root_header_decoder, +}; +use crate::adapters::filesystem_exact_record::{self as exact_record, ExactRecordError}; +use crate::adapters::{ + CatalogRestartPolicy, ChecksummedPublicationHead, FilesystemCatalogSnapshot, + filesystem_initialization_namespace, filesystem_version_two_records, publication_head_decoder, +}; +use crate::{RetentionHead, RetentionManifest, RetentionNamespaceDigest}; + +const HEAD_NAME: &str = "HEAD"; + +/// One consistent reader view: the catalog snapshot, the retention head, and +/// the manifest it selects, all observed under one shared reader fence. +/// +/// The view holds the fence for its lifetime, so collection cannot delete the +/// roots or segments it names while it lives. Selected roots are read on +/// demand and verified against the manifest's digest before they are returned. +#[must_use] +pub struct FilesystemRetentionSnapshot { + _fence: ReaderFence, + roots: Dir, + catalog: FilesystemCatalogSnapshot, + retention: Option, +} + +struct View { + catalog: FilesystemCatalogSnapshot, + retention: Option, +} + +struct Source { + root: Dir, + retention: Dir, + manifests: Dir, + store_root: PathBuf, + policy: CatalogRestartPolicy, +} + +impl RetentionViewSource for Source { + type View = View; + + fn coordinates(&mut self) -> io::Result { + let catalog = filesystem_retention_current::read_exact_optional( + &self.root, + HEAD_NAME, + publication_head_decoder::ENCODED_LENGTH, + )? + .map(|bytes| { + ChecksummedPublicationHead::decode(&bytes) + .map(|head| (head.generation(), head.catalog_digest())) + .map_err(|source| io::Error::new(io::ErrorKind::InvalidData, source)) + }) + .transpose()?; + let retention = filesystem_retention_current::observe(&self.retention, &self.manifests)? + .map(|state| (state.head().generation(), state.head().manifest_digest())); + Ok(RetentionViewCoordinates { catalog, retention }) + } + + fn load(&mut self) -> io::Result { + let catalog = FilesystemCatalogSnapshot::load(&self.store_root, self.policy) + .map_err(|source| io::Error::new(io::ErrorKind::InvalidData, source))?; + let retention = filesystem_retention_current::observe(&self.retention, &self.manifests)?; + Ok(View { catalog, retention }) + } +} + +impl FilesystemRetentionSnapshot { + /// Admits the root as version two, acquires the reader fence, and + /// double-collects one consistent view within `limit` attempts. + /// + /// The call takes no writer authority and mutates nothing. It may block + /// while collection holds the fence exclusively. + /// + /// # Errors + /// + /// Returns [`FilesystemRetentionSnapshotError`](super::FilesystemRetentionSnapshotError) + /// at the exact admission, fence, collection, or catalog refusal. + pub fn load( + store_root: &Path, + policy: CatalogRestartPolicy, + limit: ReaderAttemptLimit, + ) -> Result { + let root = Dir::open_ambient_dir(store_root, cap_std::ambient_authority()) + .map_err(|source| Error::Admission { source })?; + filesystem_initialization_namespace::admit_version_two(&root) + .map_err(|source| Error::Admission { source })?; + let _bound = filesystem_version_two_records::admit(&root) + .map_err(|source| Error::Admission { source })?; + let fence = ReaderFence::acquire(&root).map_err(|source| Error::Fence { source })?; + let retention = root + .open_dir_nofollow(pool_name::RETENTION) + .map_err(|source| Error::Admission { source })?; + let roots = retention + .open_dir_nofollow(pool_name::ROOTS) + .map_err(|source| Error::Admission { source })?; + let manifests = retention + .open_dir_nofollow(pool_name::MANIFESTS) + .map_err(|source| Error::Admission { source })?; + let mut source = Source { + root, + retention, + manifests, + store_root: store_root.to_path_buf(), + policy, + }; + let view = + collect_retention_view(&mut source, limit).map_err(|source| Error::View { source })?; + Ok(Self { + _fence: fence, + roots, + catalog: view.catalog, + retention: view.retention, + }) + } + + /// The catalog snapshot the view binds. + pub const fn catalog(&self) -> &FilesystemCatalogSnapshot { + &self.catalog + } + + /// The published retention head, or `None` when no generation is published. + #[must_use] + pub fn retention_head(&self) -> Option<&RetentionHead> { + self.retention.as_ref().map(ObservedRetentionState::head) + } + + /// The manifest the retention head selects, or `None` when none is published. + #[must_use] + pub fn manifest(&self) -> Option<&RetentionManifest> { + self.retention + .as_ref() + .map(ObservedRetentionState::manifest) + } + + /// Reads and verifies the root the manifest selects for `namespace`. + /// + /// Returns `None` when the manifest names no root for the namespace. The + /// pool entry is read without following links, bounded by the root + /// format's maximum length, decoded, and required to carry exactly the + /// generation and digest the manifest names. + /// + /// # Errors + /// + /// Returns [`FilesystemRetentionSnapshotError::Root`](super::FilesystemRetentionSnapshotError::Root) + /// when the entry is absent, unreadable, or not the selected root. + pub fn retained_root( + &self, + namespace: RetentionNamespaceDigest, + ) -> Result>, Error> { + let Some(manifest) = self.manifest() else { + return Ok(None); + }; + let entries = manifest.entries(); + let Some(entry) = entries + .binary_search_by_key(&namespace, |entry| entry.namespace()) + .ok() + .and_then(|index| entries.get(index).copied()) + else { + return Ok(None); + }; + let directory = self + .roots + .open_dir_nofollow(pool_name::namespace(namespace)) + .map_err(|source| Error::Root { source })?; + let name = pool_name::root(entry.root_generation(), entry.root_digest()); + let length = directory + .symlink_metadata(&name) + .and_then(|metadata| { + usize::try_from(metadata.len()).map_err(|_source| invalid("root length overflow")) + }) + .map_err(|source| Error::Root { source })?; + if length > root_header_decoder::MAXIMUM_ENCODED_LENGTH { + return Err(Error::Root { + source: invalid("selected root exceeds the format bound"), + }); + } + let bytes = match exact_record::read_exact_optional(&directory, &name, length) { + Ok(Some(bytes)) => bytes, + Ok(None) => { + return Err(Error::Root { + source: invalid("selected root is absent"), + }); + } + Err(ExactRecordError::Io(source)) => return Err(Error::Root { source }), + Err(ExactRecordError::Refused(refusal)) => { + return Err(Error::Root { + source: invalid_string(format!("selected root refused: {refusal}")), + }); + } + }; + let root = AdmittedRetentionRoot::decode(&bytes).map_err(|source| Error::Root { + source: io::Error::new(io::ErrorKind::InvalidData, source), + })?; + if root.digest() != entry.root_digest() + || root.root().generation() != entry.root_generation() + { + return Err(Error::Root { + source: invalid("selected root does not decode to the manifest's selection"), + }); + } + Ok(Some(bytes.into_boxed_slice())) + } +} + +fn invalid(message: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} + +fn invalid_string(message: String) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} diff --git a/src/adapters/retention/filesystem_retention_snapshot_error.rs b/src/adapters/retention/filesystem_retention_snapshot_error.rs new file mode 100644 index 0000000..876d76b --- /dev/null +++ b/src/adapters/retention/filesystem_retention_snapshot_error.rs @@ -0,0 +1,63 @@ +//! This module owns the typed error of filesystem retention snapshot loading. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::RetentionViewError; +use crate::adapters::CatalogRestartError; + +/// Why a reader could not bind one consistent version-two view. +#[derive(Debug)] +#[non_exhaustive] +pub enum FilesystemRetentionSnapshotError { + /// The root is not an exactly admitted version-two store. + Admission { + /// The exact namespace or record refusal. + source: io::Error, + }, + /// The reader fence could not be acquired. + Fence { + /// The exact filesystem failure. + source: io::Error, + }, + /// The heads never agreed, or a head read failed. + View { + /// The exact collection refusal. + source: RetentionViewError, + }, + /// The catalog `HEAD` selects a catalog that does not admit. + Catalog { + /// The exact restart refusal. + source: CatalogRestartError, + }, + /// A selected root pool entry is absent, unreadable, or not the manifest's. + Root { + /// The exact filesystem or decode refusal. + source: io::Error, + }, +} + +impl fmt::Display for FilesystemRetentionSnapshotError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Admission { .. } => "version-two reader admission refused", + Self::Fence { .. } => "reader fence acquisition failed", + Self::View { .. } => "reader view collection refused", + Self::Catalog { .. } => "catalog snapshot refused", + Self::Root { .. } => "selected retention root refused", + }) + } +} + +impl Error for FilesystemRetentionSnapshotError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Admission { source } | Self::Fence { source } | Self::Root { source } => { + Some(source) + } + Self::View { source } => Some(source), + Self::Catalog { source } => Some(source), + } + } +} diff --git a/src/adapters/retention/filesystem_retention_snapshot_tests.rs b/src/adapters/retention/filesystem_retention_snapshot_tests.rs new file mode 100644 index 0000000..8f45ae7 --- /dev/null +++ b/src/adapters/retention/filesystem_retention_snapshot_tests.rs @@ -0,0 +1,124 @@ +//! Reader fence and fenced snapshot laws over migrated stores. + +use std::error::Error; +use std::fs; + +use cap_std::fs::Dir; +use rustix::fs::{FlockOperation, flock}; + +use super::filesystem_retention_test_fixture::{ + ROOT_HEX, fixture, initial_preparation, migrated_store, open_authority, +}; +use super::{ + AdmittedRetentionRoot, FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, + ReaderAttemptLimit, ReaderFence, +}; +use crate::adapters::{ + CatalogRestartByteLimit, CatalogRestartPolicy, SegmentReadPolicy, SegmentRecordLimit, +}; +use crate::{LayoutEntryLimit, execute_retention_publication}; + +fn policy() -> Result> { + Ok(CatalogRestartPolicy::new( + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM), + CatalogRestartByteLimit::new(1_048_576)?, + )) +} + +#[test] +fn a_migrated_store_snapshot_binds_the_catalog_and_no_retention_head() -> Result<(), Box> +{ + let sandbox = migrated_store("filesystem-retention-snapshot-empty")?; + let snapshot = + FilesystemRetentionSnapshot::load(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?; + assert_eq!(snapshot.catalog().generation().get(), 1); + assert!(snapshot.retention_head().is_none()); + assert!(snapshot.manifest().is_none()); + Ok(()) +} + +#[test] +fn a_published_generation_is_read_and_its_root_verified() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-snapshot-published")?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + let _published = execute_retention_publication(&mut authority, &preparation)?; + drop(authority); + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let namespace = candidate.root().namespace().digest(); + + let snapshot = + FilesystemRetentionSnapshot::load(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?; + + let head = snapshot + .retention_head() + .ok_or("no retention head in the view")?; + assert_eq!(head.generation(), preparation.liveness_generation()); + let root = snapshot + .retained_root(namespace)? + .ok_or("the manifest does not select the published namespace")?; + assert_eq!(&*root, root_bytes.as_slice()); + Ok(()) +} + +#[test] +fn a_substituted_root_refuses_under_the_snapshot() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-snapshot-substituted")?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + let _published = execute_retention_publication(&mut authority, &preparation)?; + drop(authority); + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let path = super::filesystem_retention_test_fixture::root_pool_path(sandbox.path(), &candidate); + let mut corrupt = root_bytes.clone(); + if let Some(last) = corrupt.last_mut() { + *last ^= 0x01; + } + fs::write(&path, &corrupt)?; + + let snapshot = + FilesystemRetentionSnapshot::load(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?; + let error = snapshot + .retained_root(candidate.root().namespace().digest()) + .err() + .ok_or("a corrupt root pool entry was returned")?; + + assert!(matches!( + error, + FilesystemRetentionSnapshotError::Root { .. } + )); + Ok(()) +} + +#[test] +fn readers_share_the_fence_and_collection_cannot_take_it_exclusively() -> Result<(), Box> +{ + let sandbox = migrated_store("filesystem-retention-snapshot-fence")?; + let root = Dir::open_ambient_dir(sandbox.path(), cap_std::ambient_authority())?; + let first = ReaderFence::acquire(&root)?; + let second = ReaderFence::acquire(&root)?; + let collector = std::fs::File::open(sandbox.path().join("reader.lock"))?; + + let refused = flock(&collector, FlockOperation::NonBlockingLockExclusive); + + assert!(refused.is_err(), "an exclusive fence must wait for readers"); + drop(second); + drop(first); + flock(&collector, FlockOperation::NonBlockingLockExclusive)?; + Ok(()) +} + +#[test] +fn a_replaced_reader_lock_refuses_the_fence() -> Result<(), Box> { + let sandbox = migrated_store("filesystem-retention-snapshot-bad-fence")?; + fs::write(sandbox.path().join("reader.lock"), b"not empty")?; + let error = + FilesystemRetentionSnapshot::load(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT) + .err() + .ok_or("a non-empty reader.lock was accepted as the fence")?; + assert!(matches!( + error, + FilesystemRetentionSnapshotError::Fence { .. } + )); + Ok(()) +} diff --git a/src/adapters/retention/reader_attempt_limit.rs b/src/adapters/retention/reader_attempt_limit.rs new file mode 100644 index 0000000..67b6afd --- /dev/null +++ b/src/adapters/retention/reader_attempt_limit.rs @@ -0,0 +1,25 @@ +//! This module owns the bounded retry limit of reader view collection. + +use std::num::NonZeroU32; + +/// How many times a reader may re-collect before refusing a moving store. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[must_use] +pub struct ReaderAttemptLimit(NonZeroU32); + +impl ReaderAttemptLimit { + /// Three attempts: one publication may land between any two reads, and a + /// store that moves faster than a reader can double-collect is refused. + pub const DEFAULT: Self = Self(NonZeroU32::MIN.saturating_add(2)); + + /// Admits an explicit positive attempt count. + pub const fn new(attempts: NonZeroU32) -> Self { + Self(attempts) + } + + /// The admitted attempt count. + #[must_use] + pub const fn get(self) -> u32 { + self.0.get() + } +} diff --git a/src/adapters/retention/reader_fence.rs b/src/adapters/retention/reader_fence.rs new file mode 100644 index 0000000..966d1c9 --- /dev/null +++ b/src/adapters/retention/reader_fence.rs @@ -0,0 +1,60 @@ +//! This module owns the shared reader fence over one version-two store root. + +use std::io; + +use cap_fs_ext::MetadataExt; +use cap_std::fs::{Dir, File, Metadata}; +use rustix::fs::{FlockOperation, flock}; + +use crate::adapters::filesystem_exact_record; + +const READER_LOCK: &str = "reader.lock"; + +/// A shared kernel lock on `reader.lock` held for one snapshot's lifetime. +/// +/// Collection acquires the store writer authority and then an exclusive lock +/// on the same file, so while any fence is held no published segment, root, +/// or manifest can be deleted. Publication proceeds beside fences because it +/// only adds immutable successors. Dropping the fence releases only the +/// kernel lock; the persistent file is never deleted. +#[must_use] +pub struct ReaderFence { + _file: File, +} + +impl ReaderFence { + /// Acquires the shared fence, waiting while collection holds it exclusively. + /// + /// `reader.lock` must be a regular zero-length file reached without + /// following links; its identity is verified after the open so a swapped + /// entry refuses. + pub(super) fn acquire(root: &Dir) -> io::Result { + let file = filesystem_exact_record::open_read(root, READER_LOCK)?; + verify(root, &file)?; + flock(&file, FlockOperation::LockShared)?; + verify(root, &file)?; + Ok(Self { _file: file }) + } +} + +fn verify(root: &Dir, file: &File) -> io::Result<()> { + let handle = file.metadata()?; + let entry = root.symlink_metadata(READER_LOCK)?; + if handle.is_file() + && entry.is_file() + && handle.len() == 0 + && entry.len() == 0 + && identity(&handle) == identity(&entry) + { + Ok(()) + } else { + Err(io::Error::new( + io::ErrorKind::InvalidData, + "reader fence kind, length, or identity disagreed", + )) + } +} + +fn identity(metadata: &Metadata) -> (u64, u64) { + (metadata.dev(), metadata.ino()) +} diff --git a/src/adapters/retention/retention_view_collector.rs b/src/adapters/retention/retention_view_collector.rs new file mode 100644 index 0000000..0502e3a --- /dev/null +++ b/src/adapters/retention/retention_view_collector.rs @@ -0,0 +1,113 @@ +//! This module owns storage-independent double collection of one reader view. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::ReaderAttemptLimit; +use crate::{CatalogDigest, CatalogGeneration, LivenessGeneration, RetentionManifestDigest}; + +/// The coordinates both heads name at one instant. +/// +/// A view is accepted only when the coordinates read before loading it equal +/// the coordinates read after, so the view belongs to one catalog generation +/// and one liveness generation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RetentionViewCoordinates { + /// The catalog `HEAD` coordinate, or `None` when no catalog is published. + pub catalog: Option<(CatalogGeneration, CatalogDigest)>, + /// The `retention/HEAD` coordinate, or `None` when no retention head is published. + pub retention: Option<(LivenessGeneration, RetentionManifestDigest)>, +} + +/// The reads one reader view needs, in the order the collector calls them. +pub trait RetentionViewSource { + /// The complete view loaded between two coordinate reads. + type View; + + /// Reads both head coordinates without loading anything they select. + /// + /// # Errors + /// + /// Returns the exact read or decode failure. + fn coordinates(&mut self) -> io::Result; + + /// Loads the complete view the current heads select. + /// + /// # Errors + /// + /// Returns the exact load failure. + fn load(&mut self) -> io::Result; +} + +/// Why a reader view could not be collected. +#[derive(Debug)] +#[non_exhaustive] +pub enum RetentionViewError { + /// No catalog `HEAD` is published, so no view exists to collect. + CatalogAbsent, + /// The heads moved between every collection within the attempt limit. + AttemptsExhausted { + /// The attempts that were made. + attempts: u32, + }, + /// A read or load failed. + Io { + /// The exact failure. + source: io::Error, + }, +} + +impl fmt::Display for RetentionViewError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CatalogAbsent => formatter.write_str("no catalog head is published"), + Self::AttemptsExhausted { attempts } => write!( + formatter, + "the store moved between every one of {attempts} view collections" + ), + Self::Io { .. } => formatter.write_str("reader view collection failed"), + } + } +} + +impl Error for RetentionViewError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Io { source } => Some(source), + Self::CatalogAbsent | Self::AttemptsExhausted { .. } => None, + } + } +} + +/// Collects one view whose head coordinates agree before and after loading. +/// +/// Each attempt reads the coordinates, loads the view, and reads the +/// coordinates again; a view is accepted only when both reads agree. A +/// generation, length, digest, or checksum change discards the view and +/// retries until `limit` is exhausted, which refuses. +/// +/// # Errors +/// +/// Returns [`RetentionViewError`] for an absent catalog, an exhausted limit, +/// or the source's own failure. +pub fn collect_retention_view( + source: &mut S, + limit: ReaderAttemptLimit, +) -> Result { + let io = |source| RetentionViewError::Io { source }; + for _attempt in 0..limit.get() { + let before = source.coordinates().map_err(io)?; + if before.catalog.is_none() { + return Err(RetentionViewError::CatalogAbsent); + } + let view = source.load().map_err(io)?; + let after = source.coordinates().map_err(io)?; + if before == after { + return Ok(view); + } + } + Err(RetentionViewError::AttemptsExhausted { + attempts: limit.get(), + }) +} diff --git a/src/adapters/retention/retention_view_collector_tests.rs b/src/adapters/retention/retention_view_collector_tests.rs new file mode 100644 index 0000000..a8a9c39 --- /dev/null +++ b/src/adapters/retention/retention_view_collector_tests.rs @@ -0,0 +1,104 @@ +//! Reader view collection laws against a scripted source. + +use std::error::Error; +use std::io; +use std::num::NonZeroU32; + +use super::{ + ReaderAttemptLimit, RetentionViewCoordinates, RetentionViewError, RetentionViewSource, + collect_retention_view, +}; +use crate::{CatalogDigest, CatalogGeneration}; + +struct Scripted { + coordinates: Vec, + loads: u32, +} + +impl RetentionViewSource for Scripted { + type View = u32; + + fn coordinates(&mut self) -> io::Result { + if self.coordinates.is_empty() { + return Err(io::Error::other("script exhausted")); + } + Ok(self.coordinates.remove(0)) + } + + fn load(&mut self) -> io::Result { + self.loads = self.loads.saturating_add(1); + Ok(self.loads) + } +} + +fn published(generation: u64) -> Result> { + Ok(RetentionViewCoordinates { + catalog: Some(( + CatalogGeneration::new(generation)?, + CatalogDigest::from_validated([0; 32]), + )), + retention: None, + }) +} + +const ABSENT: RetentionViewCoordinates = RetentionViewCoordinates { + catalog: None, + retention: None, +}; + +#[test] +fn a_stable_store_is_collected_on_the_first_attempt() -> Result<(), Box> { + let mut source = Scripted { + coordinates: vec![published(1)?, published(1)?], + loads: 0, + }; + let view = collect_retention_view(&mut source, ReaderAttemptLimit::DEFAULT)?; + assert_eq!(view, 1); + Ok(()) +} + +#[test] +fn a_publication_between_the_reads_discards_the_view_and_retries() -> Result<(), Box> { + let mut source = Scripted { + coordinates: vec![published(1)?, published(2)?, published(2)?, published(2)?], + loads: 0, + }; + let view = collect_retention_view(&mut source, ReaderAttemptLimit::DEFAULT)?; + assert_eq!( + view, 2, + "the first load was discarded and the second accepted" + ); + Ok(()) +} + +#[test] +fn a_store_that_never_settles_exhausts_the_limit() -> Result<(), Box> { + let mut source = Scripted { + coordinates: (1..=8).map(published).collect::>()?, + loads: 0, + }; + let limit = ReaderAttemptLimit::new(NonZeroU32::new(2).ok_or("zero")?); + let error = collect_retention_view(&mut source, limit) + .err() + .ok_or("a moving store was accepted")?; + assert!(matches!( + error, + RetentionViewError::AttemptsExhausted { attempts: 2 } + )); + assert_eq!(source.loads, 2); + Ok(()) +} + +#[test] +fn an_absent_catalog_refuses_before_loading() -> Result<(), Box> { + let mut source = Scripted { + coordinates: vec![ABSENT], + loads: 0, + }; + let error = collect_retention_view(&mut source, ReaderAttemptLimit::DEFAULT) + .err() + .ok_or("an absent catalog was collected")?; + assert!(matches!(error, RetentionViewError::CatalogAbsent)); + assert_eq!(source.loads, 0); + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index c6ff84e..425d62a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -136,21 +136,24 @@ pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, FilesystemRetentionAuthorityError, FilesystemRetentionPublicationAuthority, - FilesystemRetentionRecoveryError, ObservedRetentionState, PreparedRetentionPublication, - RetentionAuthorityDirectory, RetentionClosureVerificationError, RetentionCurrentStateRefusal, - RetentionFixedStage, RetentionHeadDecodeError, RetentionHeadStageAssessment, - RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionManifestStageAssessment, - RetentionNamespaceAdmission, RetentionPool, RetentionPoolEntryObservation, - RetentionPoolObservations, RetentionPublicationError, RetentionPublicationOutcome, - RetentionPublicationPhase, RetentionPublicationPreparation, - RetentionPublicationPreparationError, RetentionPublicationReceipt, RetentionPublicationStorage, - RetentionRecoveryError, RetentionRecoveryEvidence, RetentionRecoveryOutcome, - RetentionRecoveryPlan, RetentionRecoveryReceipt, RetentionRecoveryRefusal, - RetentionRecoveryStep, RetentionRecoveryStorage, RetentionRootDecodeError, - RetentionRootEncodeError, RetentionRootStageAssessment, RetentionStageAssessment, - RetentionStageAssessments, RetentionTransitionDisposition, RetentionTransitionError, - RetentionTransitionPreflight, RetentionTransitionPreflightError, RetentionTransitionReadiness, - VerifiedRetentionClosure, assess_head_stage, assess_manifest_stage, assess_root_stage, + FilesystemRetentionRecoveryError, FilesystemRetentionSnapshot, + FilesystemRetentionSnapshotError, ObservedRetentionState, PreparedRetentionPublication, + ReaderAttemptLimit, ReaderFence, RetentionAuthorityDirectory, + RetentionClosureVerificationError, RetentionCurrentStateRefusal, RetentionFixedStage, + RetentionHeadDecodeError, RetentionHeadStageAssessment, RetentionManifestDecodeError, + RetentionManifestEncodeError, RetentionManifestStageAssessment, RetentionNamespaceAdmission, + RetentionPool, RetentionPoolEntryObservation, RetentionPoolObservations, + RetentionPublicationError, RetentionPublicationOutcome, RetentionPublicationPhase, + RetentionPublicationPreparation, RetentionPublicationPreparationError, + RetentionPublicationReceipt, RetentionPublicationStorage, RetentionRecoveryError, + RetentionRecoveryEvidence, RetentionRecoveryOutcome, RetentionRecoveryPlan, + RetentionRecoveryReceipt, RetentionRecoveryRefusal, RetentionRecoveryStep, + RetentionRecoveryStorage, RetentionRootDecodeError, RetentionRootEncodeError, + RetentionRootStageAssessment, RetentionStageAssessment, RetentionStageAssessments, + RetentionTransitionDisposition, RetentionTransitionError, RetentionTransitionPreflight, + RetentionTransitionPreflightError, RetentionTransitionReadiness, RetentionViewCoordinates, + RetentionViewError, RetentionViewSource, VerifiedRetentionClosure, assess_head_stage, + assess_manifest_stage, assess_root_stage, collect_retention_view, execute_retention_publication, execute_retention_recovery, plan_retention_recovery, plan_retention_transition, preflight_retention_transition, prepare_retention_publication, verify_retention_closure, From c9277eadbcb22a5ba1330760a3b908178957d4e5 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 9 Sep 2026 15:10:19 -0700 Subject: [PATCH 8/8] Test: prove retention transitions against a namespace-to-anchor-set model KEEP-RETENTION-010 asks that model operation sequences agree with a deterministic namespace-to-anchor-set map and that no caller identity, path, clock, or application policy enters the core transition. Five laws now run every three-operation sequence over initial publications of two namespaces, a successor of the first, a byte-identical retry of the last accepted publication, and an initial publication from a stale view: 125 sequences, each in a fresh migrated store, driven through the real filesystem authority. After every step the fenced reader view must equal the model exactly: the manifest's namespace-to-generation map, the liveness generation, and each selected root's generation and anchor set, with a refused operation leaving the view unchanged. The model was corrected three times by the store during development, each time toward the rule the publication page states: a byte-identical retry is already committed only while that exact staged successor, head included, remains current; a stale initial is superseded by any later publication. A source contract walks src/retention and the storage-independent retention adapters and refuses any clock, path, filesystem, environment, or identity token. KEEP-RETENTION-010 is Implemented in the ledger. Refs #19 --- CHANGELOG.md | 6 + docs/formats/segment-store-v2/README.md | 5 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/retention.rs | 2 + .../retention/retention_model_tests.rs | 363 ++++++++++++++++++ tests/retention_core_architecture_contract.rs | 61 +++ 6 files changed, 436 insertions(+), 3 deletions(-) create mode 100644 src/adapters/retention/retention_model_tests.rs create mode 100644 tests/retention_core_architecture_contract.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index da0a2f2..fa72581 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ after its public API and format compatibility policies are established. ### Added +- Model-based retention evidence: every three-operation sequence over initial + publications of two namespaces, a successor, a byte-identical retry, and a + stale initial (125 sequences, each in a fresh migrated store) agrees with a + deterministic namespace-to-(generation, anchor-set) map and liveness after + every step, observed through the fenced reader view; a source contract + keeps clocks, paths, environment, and identity out of the retention core. - `FilesystemRetentionSnapshot` is the version-two reader view: it admits the root as version two, acquires a shared `ReaderFence` on `reader.lock`, double-collects the catalog and retention heads around loading through diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 65b9e47..fb15150 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -98,9 +98,10 @@ Retention publication recovery is implemented and proven both in-process for every crash prefix and by the crash matrix, which kills a real writer before, during, and after `KEEP-CRASH-036` through `052`. Readers bind one consistent catalog, retention head, and manifest view under a -shared `ReaderFence` and verify selected roots on demand. +shared `ReaderFence` and verify selected roots on demand. Every three-operation +transition sequence agrees with a deterministic namespace-to-anchor-set model. Not implemented: partial-prefix migration recovery and `KEEP-CRASH-053..073`, -model-based transition evidence, and garbage collection. Issue #19 owns the first four and issue #21 the last; +and garbage collection. Issue #19 owns the first four and issue #21 the last; issue #97 owns the restart-stable root identity coordinate. A version-1 store remains admitted until its owner migrates it, and the [requirements ledger](requirements.md) is the authority on which requirements diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 2336887..ea326d0 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -18,7 +18,7 @@ case is not evidence. | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | recovery-required refusals before any mutation in `filesystem_retention_expectation_tests`: an absent head over populated pools, a non-initial head prepared against an absent head, an orphan directory for a namespace expected absent, and an absent directory for a namespace expected current; replaced protocol directories, an absent or changed head-selected catalog, an over-full census, zero-generation pool names, and a stage retained by a failed write refuse in `filesystem_retention_*_tests`; storage-independent classification of every fixed-stage crash prefix (discard, link and protect, finalize, clean up, or typed refusal) in `recovery_planner_tests`; every publication prefix 0 through 18, each mid-write truncation, and successor prefixes recover in-process to the documented state, idempotently, with the forward retry reporting the predicted outcome, in `filesystem_retention_recovery_prefix_tests`; `cargo xtask durability-crash-matrix` kills a real writer before, during, and after `KEEP-CRASH-036` through `052` and requires restart recovery to reach the documented state and the forward retry to report the predicted outcome | Implemented | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | `ReaderFence` holds a shared kernel lock on a verified zero-length `reader.lock`; `collect_retention_view` accepts a view only when both head coordinates agree before and after loading and refuses an exhausted attempt limit (`retention_view_collector_tests`); `FilesystemRetentionSnapshot` binds the catalog snapshot, retention head, and manifest under the fence and verifies each selected root on demand while the fence is held, refusing a substituted root and a replaced fence, and two readers share the fence while an exclusive lock waits (`filesystem_retention_snapshot_tests`) | Implemented | | `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests`; committed retry reopens the head-selected manifest entry and root pool bytes, refusing absent, changed, or corrupt evidence in `filesystem_retention_current_tests`; every refusal is a typed `RetentionCurrentStateRefusal` source, with superseded, committed-root-absent, committed-root-changed, and head-absent-with-artifacts pinned by downcast | Implemented | -| `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | model-based and source-architecture tests | Planned in #19 | +| `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | every three-operation sequence over initial publications of two namespaces, a successor, a byte-identical retry, and a stale initial (125 sequences, each in a fresh migrated store) agrees with a deterministic namespace-to-(generation, anchor-set) map plus liveness after every step, observed through the fenced reader view, in `retention_model_tests`; `tests/retention_core_architecture_contract.rs` refuses any clock, path, environment, or identity token in the retention core | Implemented | diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 84b6c29..606ce05 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -115,6 +115,8 @@ mod recovery_planner_tests; mod recovery_refusal; mod recovery_stage_assessment; mod recovery_storage; +#[cfg(test)] +mod retention_model_tests; mod retention_view_collector; #[cfg(test)] mod retention_view_collector_tests; diff --git a/src/adapters/retention/retention_model_tests.rs b/src/adapters/retention/retention_model_tests.rs new file mode 100644 index 0000000..55c7ef8 --- /dev/null +++ b/src/adapters/retention/retention_model_tests.rs @@ -0,0 +1,363 @@ +//! Model-based retention laws: every operation sequence agrees with a +//! deterministic namespace-to-anchor-set map, and a refused operation leaves +//! the fenced reader view exactly where it was. + +use std::collections::BTreeMap; +use std::error::Error; +use std::path::PathBuf; + +use super::filesystem_retention_test_fixture::{ + ROOT_HEX, fixture, initial_preparation, initial_root, new_namespace_preparation, + open_authority, successor_preparation, successor_root, +}; +use super::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, FilesystemRetentionPublicationAuthority, + FilesystemRetentionSnapshot, ReaderAttemptLimit, RetentionPublicationOutcome, + RetentionPublicationPreparation, +}; +use crate::adapters::{ + CatalogRestartByteLimit, CatalogRestartPolicy, SegmentReadPolicy, SegmentRecordLimit, +}; +use crate::{ + LayoutEntryLimit, RetentionAnchor, RetentionNamespaceDigest, execute_retention_publication, +}; + +const NAMESPACE_B: &[u8] = b"model-namespace-b"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Namespace { + A, + B, +} + +#[derive(Clone, Copy, Debug)] +enum Operation { + /// Publish generation one of the namespace from a fresh view. + Initial(Namespace), + /// Publish the exact successor of namespace A from a fresh view. + Successor, + /// Replay the last accepted publication byte for byte. + RetryLast, + /// Publish generation one of namespace A from a view that predates it. + StaleInitial, +} + +const OPERATIONS: [Operation; 5] = [ + Operation::Initial(Namespace::A), + Operation::Initial(Namespace::B), + Operation::Successor, + Operation::RetryLast, + Operation::StaleInitial, +]; + +/// The byte ingredients of one preparation; rebuilding it is byte-identical. +#[derive(Clone)] +enum Recipe { + Initial { + candidate: Vec, + manifest: Option>, + }, + Successor { + current_root: Vec, + manifest: Vec, + candidate: Vec, + }, +} + +impl Recipe { + fn candidate(&self) -> &[u8] { + match self { + Self::Initial { candidate, .. } | Self::Successor { candidate, .. } => candidate, + } + } + + fn publish( + &self, + authority: &mut FilesystemRetentionPublicationAuthority, + ) -> Result> { + let preparation: RetentionPublicationPreparation<'_> = match self { + Self::Initial { + candidate, + manifest: None, + } => initial_preparation(candidate)?, + Self::Initial { + candidate, + manifest: Some(manifest), + } => { + new_namespace_preparation(&AdmittedRetentionManifest::decode(manifest)?, candidate)? + } + Self::Successor { + current_root, + manifest, + candidate, + } => successor_preparation( + &AdmittedRetentionRoot::decode(current_root)?, + &AdmittedRetentionManifest::decode(manifest)?, + candidate, + )?, + }; + Ok(execute_retention_publication(authority, &preparation)?.outcome()) + } +} + +/// The reference model: namespace digest to (generation, anchors), plus the +/// liveness generation, which counts accepted publications. +#[derive(Default)] +struct Model { + namespaces: BTreeMap)>, + liveness: u64, +} + +struct Store { + authority: FilesystemRetentionPublicationAuthority, + path: PathBuf, + template: Vec, + last_accepted: Option, +} + +fn policy() -> Result> { + Ok(CatalogRestartPolicy::new( + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM), + CatalogRestartByteLimit::new(1_048_576)?, + )) +} + +fn snapshot(store: &Store) -> Result> { + Ok(FilesystemRetentionSnapshot::load( + &store.path, + policy()?, + ReaderAttemptLimit::DEFAULT, + )?) +} + +fn digest_of(bytes: &[u8]) -> Result> { + Ok(AdmittedRetentionRoot::decode(bytes)? + .root() + .namespace() + .digest()) +} + +fn candidate_bytes(store: &Store, namespace: Namespace) -> Result, Box> { + match namespace { + Namespace::A => Ok(store.template.clone()), + Namespace::B => { + let template = AdmittedRetentionRoot::decode(&store.template)?; + Ok(initial_root(NAMESPACE_B, &template)?.encoded().to_vec()) + } + } +} + +/// A recipe and the answer the model expects for it. +type Planned = (Recipe, Expected); + +/// What the model says the store must answer. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Expected { + Published, + AlreadyCommitted, + Refused, +} + +/// Builds the recipe an operation would publish and the model's expected +/// answer. `None` means the operation has no candidate. +fn recipe( + store: &Store, + model: &Model, + operation: Operation, +) -> Result, Box> { + let fresh_manifest = store + .authority + .observe_current()? + .map(|state| state.manifest_bytes().to_vec()); + Ok(match operation { + Operation::Initial(namespace) => { + let candidate = candidate_bytes(store, namespace)?; + let expected = if model.namespaces.contains_key(&digest_of(&candidate)?) { + Expected::Refused + } else { + Expected::Published + }; + Some(( + Recipe::Initial { + candidate, + manifest: fresh_manifest, + }, + expected, + )) + } + Operation::StaleInitial => { + // The stale caller's preparation stages liveness generation one, + // so it is byte-identical to the accepted publication only while + // that publication is still the whole history; any later + // publication supersedes it. + let candidate = candidate_bytes(store, Namespace::A)?; + let expected = match ( + model.namespaces.get(&digest_of(&candidate)?), + model.liveness, + ) { + (None, 0) => Expected::Published, + (Some(_), 1) => Expected::AlreadyCommitted, + _ => Expected::Refused, + }; + Some(( + Recipe::Initial { + candidate, + manifest: None, + }, + expected, + )) + } + Operation::Successor => { + let digest = digest_of(&store.template)?; + if !model.namespaces.contains_key(&digest) { + return Ok(None); + } + let current_root = snapshot(store)? + .retained_root(digest)? + .ok_or("model root absent on disk")? + .to_vec(); + let candidate = successor_root(&AdmittedRetentionRoot::decode(¤t_root)?)? + .encoded() + .to_vec(); + Some(( + Recipe::Successor { + current_root, + manifest: fresh_manifest.ok_or("successor over no manifest")?, + candidate, + }, + Expected::Published, + )) + } + Operation::RetryLast => store + .last_accepted + .clone() + .map(|recipe| (recipe, Expected::AlreadyCommitted)), + }) +} + +/// Applies one operation to the store and the model. +fn apply(store: &mut Store, model: &mut Model, operation: Operation) -> Result<(), Box> { + let Some((recipe, expected)) = recipe(store, model, operation)? else { + return Ok(()); + }; + match (expected, recipe.publish(&mut store.authority)) { + (Expected::AlreadyCommitted, Ok(RetentionPublicationOutcome::AlreadyCommitted)) + | (Expected::Refused, Err(_)) => {} + (Expected::Published, Ok(RetentionPublicationOutcome::Published)) => { + let candidate = AdmittedRetentionRoot::decode(recipe.candidate())?; + model.namespaces.insert( + candidate.root().namespace().digest(), + ( + candidate.root().generation().get(), + candidate.root().anchors().to_vec(), + ), + ); + model.liveness = model.liveness.saturating_add(1); + store.last_accepted = Some(recipe); + } + (expected, result) => { + return Err( + format!("{operation:?}: model expects {expected:?}, store {result:?}").into(), + ); + } + } + Ok(()) +} + +/// Requires the fenced reader view to agree with the model exactly. +fn verify(store: &Store, model: &Model) -> Result<(), Box> { + let snapshot = snapshot(store)?; + let observed: BTreeMap<_, _> = snapshot + .manifest() + .map(|manifest| { + manifest + .entries() + .iter() + .map(|entry| (entry.namespace(), entry.root_generation().get())) + .collect() + }) + .unwrap_or_default(); + let expected: BTreeMap<_, _> = model + .namespaces + .iter() + .map(|(namespace, (generation, _))| (*namespace, *generation)) + .collect(); + assert_eq!(observed, expected, "manifest disagrees with the model"); + let liveness = snapshot + .retention_head() + .map_or(0, |head| head.generation().get()); + assert_eq!( + liveness, model.liveness, + "liveness generation disagrees with the model" + ); + for (namespace, (generation, anchors)) in &model.namespaces { + let bytes = snapshot + .retained_root(*namespace)? + .ok_or("model namespace has no root on disk")?; + let root = AdmittedRetentionRoot::decode(&bytes)?; + assert_eq!(root.root().generation().get(), *generation); + assert_eq!( + root.root().anchors(), + anchors.as_slice(), + "anchor set disagrees with the model" + ); + } + Ok(()) +} + +/// Runs every three-operation sequence that starts with `first` in a fresh +/// migrated store each, checking the fenced view against the model after +/// every step. +fn run_sequences(first: Operation, label: &str) -> Result<(), Box> { + let template = fixture(ROOT_HEX)?; + let mut sequences = 0_u32; + for second in OPERATIONS { + for third in OPERATIONS { + let name = format!("filesystem-retention-model-{label}-{sequences}"); + let (sandbox, authority) = open_authority(&name)?; + let mut store = Store { + authority, + path: sandbox.path().to_path_buf(), + template: template.clone(), + last_accepted: None, + }; + let mut model = Model::default(); + for operation in [first, second, third] { + apply(&mut store, &mut model, operation) + .map_err(|error| format!("{first:?} {second:?} {third:?}: {error}"))?; + verify(&store, &model) + .map_err(|error| format!("{first:?} {second:?} {third:?}: {error}"))?; + } + sequences = sequences.saturating_add(1); + } + } + assert_eq!(sequences, 25); + Ok(()) +} + +#[test] +fn sequences_starting_with_an_initial_publication_of_a_agree_with_the_model() +-> Result<(), Box> { + run_sequences(Operation::Initial(Namespace::A), "initial-a") +} + +#[test] +fn sequences_starting_with_an_initial_publication_of_b_agree_with_the_model() +-> Result<(), Box> { + run_sequences(Operation::Initial(Namespace::B), "initial-b") +} + +#[test] +fn sequences_starting_with_a_successor_agree_with_the_model() -> Result<(), Box> { + run_sequences(Operation::Successor, "successor") +} + +#[test] +fn sequences_starting_with_a_retry_agree_with_the_model() -> Result<(), Box> { + run_sequences(Operation::RetryLast, "retry") +} + +#[test] +fn sequences_starting_with_a_stale_initial_agree_with_the_model() -> Result<(), Box> { + run_sequences(Operation::StaleInitial, "stale") +} diff --git a/tests/retention_core_architecture_contract.rs b/tests/retention_core_architecture_contract.rs new file mode 100644 index 0000000..3b75430 --- /dev/null +++ b/tests/retention_core_architecture_contract.rs @@ -0,0 +1,61 @@ +//! The retention core admits no caller identity, path, clock, or application policy. + +use std::error::Error; +use std::fs; +use std::path::Path; + +/// Tokens that would let identity, paths, clocks, or environment into a +/// transition decision. The filesystem adapters own paths; the core does not. +const FORBIDDEN: [&str; 8] = [ + "SystemTime", + "Instant", + "std::env", + "std::path", + "std::fs", + "getuid", + "hostname", + "username", +]; + +/// Storage-independent retention modules outside `src/retention/`. +const CORE_ADAPTERS: [&str; 8] = [ + "src/adapters/retention/transition_planner.rs", + "src/adapters/retention/transition_preflight.rs", + "src/adapters/retention/publication_preparation.rs", + "src/adapters/retention/publication_execution.rs", + "src/adapters/retention/publication_storage.rs", + "src/adapters/retention/recovery_planner.rs", + "src/adapters/retention/recovery_execution.rs", + "src/adapters/retention/retention_view_collector.rs", +]; + +#[test] +fn the_retention_core_admits_no_identity_path_clock_or_policy() -> Result<(), Box> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut sources = Vec::new(); + for entry in fs::read_dir(root.join("src/retention"))? { + let path = entry?.path(); + if path.extension().is_some_and(|extension| extension == "rs") { + sources.push(path); + } + } + for adapter in CORE_ADAPTERS { + let path = root.join(adapter); + assert!( + path.is_file(), + "{adapter} is missing; update the contract list" + ); + sources.push(path); + } + for path in sources { + let source = fs::read_to_string(&path)?; + for token in FORBIDDEN { + assert!( + !source.contains(token), + "{} names `{token}`; the retention core decides from evidence alone", + path.display() + ); + } + } + Ok(()) +}