diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 7b78b4f..fa3c700 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -34,7 +34,6 @@ export default withMermaid({ { text: 'OpenStack', link: '/openstack/' }, { text: 'Storage', link: '/storage/' }, { text: 'Networking', link: '/networking/' }, - { text: 'Observability', link: '/observability/' }, { text: 'Management', link: '/management/' }, { text: 'API Reference', link: '/api/' }, ], @@ -128,16 +127,6 @@ export default withMermaid({ useFolderTitleFromIndexFile: true, sortMenusByFrontmatterOrder: true, }, - { - documentRootPath: '/docs/', - scanStartPath: 'observability', - resolvePath: '/observability/', - capitalizeFirst: false, - useTitleFromFrontmatter: true, - useFolderLinkFromIndexFile: true, - useFolderTitleFromIndexFile: true, - sortMenusByFrontmatterOrder: true, - }, { documentRootPath: '/docs/', scanStartPath: 'management', diff --git a/docs/getting-started/full-deployment.md b/docs/getting-started/full-deployment.md index 4e5542c..7de5e0d 100644 --- a/docs/getting-started/full-deployment.md +++ b/docs/getting-started/full-deployment.md @@ -80,9 +80,9 @@ Follow the [Minimal Environment](/getting-started/minimal) guide first to unders ### Observability layer -- [ ] Prysm deployed and collecting events - see [Observability - Prysm](/observability/prysm) -- [ ] Prometheus scraping all targets - see [Observability - Prometheus](/observability/prometheus) -- [ ] Perses dashboards deployed - see [Observability - Perses](/observability/perses) +- [ ] Prysm deployed and collecting events - see [Observability - Prysm](/storage/observability/prysm) +- [ ] Prometheus scraping all targets - see [Observability - Prometheus](/storage/observability/prometheus) +- [ ] Perses dashboards deployed - see [Observability - Perses](/storage/observability/perses) - [ ] Alerting rules configured and routing verified ### Management layer diff --git a/docs/getting-started/minimal.md b/docs/getting-started/minimal.md index ed7a2bf..27b098a 100644 --- a/docs/getting-started/minimal.md +++ b/docs/getting-started/minimal.md @@ -90,10 +90,93 @@ helm repo update helm install rook-ceph rook-release/rook-ceph \ --namespace rook-ceph \ - --create-namespace + --create-namespace \ + --version v1.20.7 \ + --wait + +helm repo add ceph-csi-operator https://ceph.github.io/ceph-csi-operator +helm repo update +helm install ceph-csi-drivers ceph-csi-operator/ceph-csi-drivers \ + --namespace rook-ceph \ + --version 1.0.4 \ + --wait \ + -f https://raw.githubusercontent.com/rook/rook/v1.20.7/deploy/charts/ceph-csi-drivers/values.yaml +``` + +Create `minimal-ceph-cluster.yaml`, replacing `k3s-node` and `sdb` with the +Kubernetes node name and the dedicated raw device in your lab: + +```yaml +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + name: rook-ceph + namespace: rook-ceph +spec: + dataDirHostPath: /var/lib/rook + cephVersion: + image: quay.io/ceph/ceph:v20.2.4 + mon: + count: 1 + allowMultiplePerNode: true + mgr: + count: 1 + allowMultiplePerNode: true + storage: + useAllNodes: false + useAllDevices: false + nodes: + - name: k3s-node + devices: + - name: sdb + cephConfig: + global: + osd_pool_default_size: "1" + mon_warn_on_pool_no_redundancy: "false" +--- +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: replicapool + namespace: rook-ceph +spec: + replicated: + size: 1 + requireSafeReplicaSize: false +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: rook-ceph-block +provisioner: rook-ceph.rbd.csi.ceph.com +parameters: + clusterID: rook-ceph + pool: replicapool + imageFormat: "2" + imageFeatures: layering + csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph + csi.storage.k8s.io/controller-publish-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/controller-publish-secret-namespace: rook-ceph + csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node + csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph + csi.storage.k8s.io/fstype: ext4 +allowVolumeExpansion: true +reclaimPolicy: Delete +``` + +The device must be unformatted, unmounted, and dedicated to this disposable +test cluster. Apply the manifest and wait for Ceph to become ready: + +```bash +kubectl apply -f minimal-ceph-cluster.yaml +kubectl -n rook-ceph wait --for=condition=Ready cephcluster/rook-ceph --timeout=15m ``` -Then deploy a minimal single-node `CephCluster`. See [Storage - Rook](/storage/rook) for the full cluster configuration. +This single-node configuration is for testing only. See [Storage - Rook](/storage/rook) +for the production three-node cluster configuration. ## Step 5: Deploy OpenStack diff --git a/docs/management/greenhouse.md b/docs/management/greenhouse.md index 6695459..0f02b31 100644 --- a/docs/management/greenhouse.md +++ b/docs/management/greenhouse.md @@ -22,4 +22,4 @@ Greenhouse aggregates operational views from multiple tools - Prometheus, Perses - [Greenhouse documentation](https://cloudoperators.github.io/greenhouse/) - [Greenhouse GitHub](https://github.com/cloudoperators/greenhouse) -- [Observability](/observability/) - Prometheus and Perses, which Greenhouse surfaces +- [Observability](/storage/observability/) - Prometheus and Perses, which Greenhouse surfaces diff --git a/docs/overview/architecture.md b/docs/overview/architecture.md index 9d265e8..627806a 100644 --- a/docs/overview/architecture.md +++ b/docs/overview/architecture.md @@ -42,7 +42,9 @@ The infrastructure data plane - the raw execution environment for virtual machin ## Observability -Cross-cutting. [Prysm](/observability/prysm) collects audit and operational events. [Prometheus](/observability/prometheus) scrapes metrics from all layers. [Perses](/observability/perses) provides dashboards. +Cross-cutting. [Prysm](/storage/observability/prysm) collects audit and operational +events. [Prometheus](/storage/observability/prometheus) scrapes metrics from all +layers. [Perses](/storage/observability/perses) provides dashboards. ## OpenStack diff --git a/docs/overview/index.md b/docs/overview/index.md index e91fab4..ddea7e2 100644 --- a/docs/overview/index.md +++ b/docs/overview/index.md @@ -36,4 +36,4 @@ CobaltCore is for teams that: - **[Architecture](/overview/architecture)** - understand how the layers fit together - **[Why CobaltCore](/overview/why)** - the reasoning behind the design decisions - **[Showroom](/showroom/)** - live examples and demos of CobaltCore in action -- Layer sections (**Platform**, **Compute**, **OpenStack**, **Storage**, **Networking**, **Observability**, **Management**) - deep-dive reference for each part of the stack +- Layer sections (**Platform**, **Compute**, **OpenStack**, **Storage**, **Networking**, **Management**) - deep-dive reference for each part of the stack. Storage includes the observability documentation. diff --git a/docs/storage/arbiter.md b/docs/storage/arbiter.md index e3781db..3db9e84 100644 --- a/docs/storage/arbiter.md +++ b/docs/storage/arbiter.md @@ -6,12 +6,20 @@ outline: deep # Arbiter +## About This Project + +The external-arbiter-operator (Arbiter) works with Rook-provisioned Ceph +clusters to deploy external arbiters (monitors) that are not managed by Rook +but that participate in consensus. + +The operator also monitors the remote cluster to verify its availability and +ensure that the tenant has sufficient permissions to handle the deployment of +Arbiter. + ::: tip Source Code [github.com/cobaltcore-dev/external-arbiter-operator](https://github.com/cobaltcore-dev/external-arbiter-operator) ::: -Arbiter deploys external Ceph monitors at a tiebreaker site, enabling Ceph clusters to maintain quorum across two data centers. Without Arbiter, a two-site cluster loses quorum whenever one site becomes unavailable - Arbiter adds a lightweight third monitor at a separate site to break the tie. - ## Why Arbiter? In a stretched Ceph cluster spanning two availability zones (AZ-A and AZ-B), monitors are split evenly between the two sites. If either AZ loses connectivity, neither side can reach a majority - the cluster is unable to make quorum decisions and becomes unavailable. @@ -26,7 +34,8 @@ Arbiter restores **control-plane quorum** - it does not guarantee data redundanc ## How it works -The Arbiter operator watches the Ceph cluster for quorum loss. When quorum is lost between the two primary sites, the operator: +The Arbiter operator provisions the external monitor before a failure occurs and +continuously reconciles its desired state. During reconciliation, the operator: 1. Reads the current Ceph cluster state 2. Reserves an external monitor ID @@ -35,52 +44,209 @@ The Arbiter operator watches the Ceph cluster for quorum loss. When quorum is lo The operator also continuously monitors the remote cluster to verify its availability and ensure the necessary RBAC permissions are in place. -## Deployment +## Requirements and Setup -### Prerequisites +### Required Tools + +The following tools are required on your development machine: + +- `sed` +- `openssl` +- `make` +- `git` +- `golang` +- `lima` +- `kubectl` +- `nerdctl` (available inside the Lima VM) +- `helm` + +The remaining dependencies are provisioned via Go tools, including the +Kubebuilder toolset. -- A Ceph cluster managed by Rook running on the primary cluster -- A second (tiebreaker) Kubernetes cluster reachable from the primary cluster -- `cert-manager` installed on the primary cluster +## Quick Start -### Install +What follows is a quick walkthrough on how to prepare the environment, run the +operator locally, and deploy an external monitor. + +::: warning Development topology +This quick start uses one Lima Kubernetes cluster for local development. It +does not validate the failure isolation of a production stretched cluster, +where the arbiter monitor must run in a separate failure domain. +::: + +### Clone and Setup ```bash -helm install --create-namespace --namespace arbiter-operator \ - --values values.yaml \ - arbiter-operator oci://ghcr.io/cobaltcore-dev/charts/external-arbiter-operator +# Clone the Arbiter repository +git clone https://github.com/cobaltcore-dev/external-arbiter-operator.git +cd external-arbiter-operator + +# Install the development dependencies +make deps + +# Clone the Rook manifests used below +git clone --depth 1 --branch v1.20.7 https://github.com/rook/rook.git + +# Create OSD for Ceph +limactl disk create osd --size=8G + +# Create VM instance +limactl create --name=k8s ./contrib/vm.yaml + +# Start VM +limactl start k8s + +# Use kubeconfig provided by VM +export KUBECONFIG="${HOME}/.lima/k8s/copied-from-guest/kubeconfig.yaml" +``` + +### Install Prerequisites + +```bash +# Install cert-manager +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.21.1/cert-manager.yaml + +# Install Rook operator +kubectl apply -f ./rook/deploy/examples/crds.yaml +kubectl apply -f ./rook/deploy/examples/common.yaml +kubectl apply -f ./rook/deploy/examples/operator.yaml +kubectl apply -f ./rook/deploy/examples/csi-operator.yaml + +# Create Ceph cluster +kubectl apply -f ./rook/deploy/examples/cluster-test.yaml +kubectl -n rook-ceph wait --for=condition=Ready cephcluster --all --timeout=15m + +# Install the Ceph toolbox used by the verification step +kubectl apply -f ./rook/deploy/examples/toolbox.yaml +kubectl -n rook-ceph rollout status deployment/rook-ceph-tools --timeout=10m ``` -### Configure +### Build and Install Operator -Create the remote cluster and arbiter resources: +```bash +# Build image +limactl shell k8s sudo nerdctl --namespace k8s.io build \ + -t localhost:5000/cobaltcore-dev/external-arbiter-operator:latest \ + -f ./Dockerfile . + +# Dry run operator install via Helm +helm install --dry-run --create-namespace --namespace arbiter-operator \ + --values ./contrib/charts/external-arbiter-operator/local.yaml \ + arbiter-operator ./contrib/charts/external-arbiter-operator + +# Install operator via Helm chart +helm install --create-namespace --namespace arbiter-operator \ + --values ./contrib/charts/external-arbiter-operator/local.yaml \ + arbiter-operator ./contrib/charts/external-arbiter-operator +``` + +### Configure and Deploy Arbiter ```bash -# Create the secret with kubeconfig for the tiebreaker cluster -kubectl apply -f secret.yaml -n arbiter-operator +# Create namespace, user, role, rolebinding, kubeconfig and secret for arbiter +./hack/configure-k8s-user.sh + +# Create secret with remote cluster access configuration +kubectl apply -f ./contrib/k8s/examples/secret.yaml -n arbiter-operator -# Register the tiebreaker cluster -kubectl apply -f remote-cluster.yaml -n arbiter-operator +# Create remote cluster resource +kubectl apply -f ./contrib/k8s/examples/remote-cluster.yaml -n arbiter-operator -# Deploy the arbiter monitor -kubectl apply -f remote-arbiter.yaml -n arbiter-operator +# Create remote arbiter resource +kubectl apply -f ./contrib/k8s/examples/remote-arbiter.yaml -n arbiter-operator -# Watch until ready +# Watch until Arbiter is ready, then press Ctrl+C kubectl get remotearbiter -n arbiter-operator -w -# Verify the arbiter joined quorum +# Check that Arbiter has joined quorum kubectl exec deployment/rook-ceph-tools -n rook-ceph -it -- ceph mon dump ``` -### CRDs +### Cleanup + +```bash +# Delete managed resources while the operator can still finalize them +kubectl delete -f ./contrib/k8s/examples/remote-arbiter.yaml -n arbiter-operator +kubectl delete -f ./contrib/k8s/examples/remote-cluster.yaml -n arbiter-operator +kubectl delete -f ./contrib/k8s/examples/secret.yaml -n arbiter-operator + +# Remove Helm chart +helm uninstall --namespace arbiter-operator arbiter-operator + +# Stop VM +limactl stop k8s + +# Delete VM +limactl delete k8s + +# Delete the detached OSD disk +limactl disk delete osd +``` + +## Make Goals + +Useful make commands for development: + +```bash +# Build binary +make + +# Prettify project, run linters, etc. +make pretty + +# Run tests +make test + +# Regenerate Kubernetes resources +make gen + +# Copy CRD definitions to Helm chart +make helm +``` + +## Configuration + +### Deployment Configuration + +Deployment manifests are managed by Helm. The `values.yaml` file lists all +available configuration options. + +### Resource Configuration + +The following example resources are provided: + +- `secret.yaml` - Kubeconfig secret for arbiter installation +- `remote-cluster.yaml` - RemoteCluster resource definition +- `remote-arbiter.yaml` - RemoteArbiter resource definition + +## How to Run + +### Prerequisites + +Before running the operator, ensure the following conditions are met: + +1. A Ceph cluster operated by Rook is already up and running on the source + Kubernetes cluster +1. Resources (pods, services) from the target (arbiter) cluster are reachable + from the source (operator/Rook) cluster and vice versa + +### Deployment Steps -| Resource | Role | -|---|---| -| `RemoteCluster` | Represents the tiebreaker Kubernetes cluster; references the kubeconfig secret | -| `RemoteArbiter` | Triggers deployment of the external monitor on the `RemoteCluster` | +1. Create a user on the target cluster. +1. Create the target namespace on the target cluster. +1. Grant the user permissions to manage deployments, secrets, configmaps, their + statuses, and finalizers. +1. Provision the target user kubeconfig on the source cluster via secret. +1. Deploy the operator on the source cluster. +1. Create a RemoteCluster resource on the source cluster, referencing the target + user kubeconfig secret. +1. Create a RemoteArbiter resource on the source cluster, referencing the + RemoteCluster. +1. Watch until resources are ready. +1. Verify that the arbiter has joined the quorum by running `ceph mon dump`. ## See also -- [Storage - Ceph](./ceph) -- [Storage - Rook](./rook) -- [Arbiter GitHub repository](https://github.com/cobaltcore-dev/external-arbiter-operator) +- [Storage - Ceph](./ceph.md) +- [Storage - Rook](./rook.md) +- [Arbiter project repository](https://github.com/cobaltcore-dev/external-arbiter-operator?tab=readme-ov-file) diff --git a/docs/storage/ceph.md b/docs/storage/ceph.md index af134fa..3db5b2c 100644 --- a/docs/storage/ceph.md +++ b/docs/storage/ceph.md @@ -6,7 +6,14 @@ outline: deep # Ceph -Ceph is the distributed storage engine at the foundation of CobaltCore's storage layer. It provides three storage interfaces - block (RBD), file (CephFS), and object (RGW) - from a single cluster, managed by [Rook](./rook) as a Kubernetes-native workload. +Ceph is a clustered and distributed storage manager. + +Ceph uniquely delivers object, block, and file storage in one unified system. +Ceph is highly reliable, easy to manage, and free. Ceph delivers extraordinary +scalability - thousands of clients accessing petabytes to exabytes of data. A +Ceph Node leverages commodity hardware and intelligent daemons, and a Ceph +Storage Cluster accommodates large numbers of nodes, which communicate with +each other to replicate and redistribute data dynamically. ## How CobaltCore uses Ceph @@ -57,8 +64,451 @@ Ceph uses the CRUSH algorithm to determine where data is stored across OSDs - no CobaltCore uses **replication** (typically 3 copies) for most pools. Erasure coding is available for large object stores where storage efficiency matters more than write latency. -## See also +## Storage interfaces and data services + +### Ceph Block Device Summary (RBD) + +#### Overview of RBD + +A block is a sequence of bytes, often 512 bytes in size. Block-based storage +interfaces represent a mature and common method for storing data on various +media types including hard disk drives (HDDs), solid-state drives (SSDs), +compact discs (CDs), floppy disks, and magnetic tape. The widespread adoption +of block device interfaces makes them an ideal fit for mass data storage +applications, including their integration with Ceph storage systems. + +#### Core Features + +Ceph block devices are designed with three fundamental characteristics: +thin-provisioning, resizability, and data striping across multiple Object +Storage Daemons (OSDs). These devices leverage the full capabilities of RADOS +(Reliable Autonomic Distributed Object Store), including snapshotting, +replication, and strong consistency guarantees. Ceph block storage clients +establish communication with Ceph clusters through two primary methods: kernel +modules or the librbd library. + +An important distinction exists between these two communication methods +regarding caching behavior. Kernel modules have the capability to utilize Linux +page caching for performance optimization. For applications that rely on the +librbd library, Ceph provides its own RBD (RADOS Block Device) caching +mechanism to enhance performance. + +#### Performance and Scalability + +Ceph's block devices are engineered to deliver high performance combined with +vast scalability capabilities. This performance extends to various deployment +scenarios, including direct integration with kernel modules and virtualization +environments. The architecture supports virtualization stacks such as QEMU/KVM, +enabling efficient virtualized storage operations. + +Cloud-based computing platforms have embraced Ceph block devices as a storage +backend solution. Major cloud computing systems including OpenStack, OpenNebula, +and CloudStack integrate with Ceph block devices through their reliance on +libvirt and QEMU technologies. This integration allows these cloud platforms to +leverage Ceph's distributed storage capabilities for their virtual machine +storage requirements. + +#### Unified Storage Cluster + +One of Ceph's significant architectural advantages is its ability to support +multiple storage interfaces simultaneously within a single cluster. The same +Ceph cluster can concurrently operate the Ceph RADOS Gateway for object +storage, the Ceph File System (CephFS) for file-based storage, and Ceph block +devices for block-based storage. This unified approach eliminates the need for +separate storage infrastructure for different storage paradigms, simplifying +management and reducing operational overhead. + +This multi-interface capability allows organizations to deploy a single storage +solution that addresses diverse storage requirements, from traditional block +storage for databases and virtual machines to object storage for unstructured +data and file storage for shared filesystems. The convergence of these storage +types within one cluster provides operational efficiency and cost-effectiveness +while maintaining the performance and reliability characteristics required for +enterprise deployments. + +#### Technical Implementation + +The thin-provisioning feature of Ceph block devices means that storage space is +allocated only as data is written, rather than pre-allocating the entire volume +capacity upfront. This approach optimizes storage utilization by avoiding waste +from unused pre-allocated space and allows for oversubscription strategies +where the sum of provisioned capacity can exceed physical capacity, based on +actual usage patterns. + +The resizable nature of Ceph block devices provides operational flexibility. +Administrators can expand images as requirements grow. Shrinking an image is a +separate, potentially destructive operation that requires the contained +filesystem to be reduced first and explicit confirmation to `rbd resize`. + +Data striping across multiple OSDs distributes data blocks across the cluster's +storage nodes. This distribution achieves two critical objectives: it increases +aggregate throughput by allowing parallel I/O operations across multiple +devices, and it ensures data availability through the replication mechanisms +built into RADOS. The striping process breaks data into smaller chunks that are +distributed according to the cluster's CRUSH (Controlled Scalable Decentralized +Placement of Replicated Data) algorithm, which determines optimal placement +based on cluster topology and configured policies. + +#### RADOS Integration + +The integration with RADOS provides Ceph block devices with enterprise-grade +features. Snapshotting capability enables point-in-time copies of block devices, +supporting backup operations, testing scenarios, and recovery procedures. +Snapshots are space-efficient, storing only changed data rather than full +copies. Applications should still coordinate writes when an +application-consistent snapshot is required. + +Replication ensures data durability by maintaining multiple copies of data +across different cluster nodes. The replication factor is configurable, +allowing organizations to balance storage efficiency against data protection +requirements. Strong consistency guarantees ensure that all replicas reflect the +same data state, preventing split-brain scenarios and ensuring data integrity +even during failure conditions. + +The communication architecture between block storage clients and Ceph clusters +through kernel modules or librbd provides flexibility in deployment scenarios. +Kernel module integration enables direct access from operating systems, while +librbd allows applications to interact with Ceph block devices programmatically, +supporting a wide range of use cases from bare-metal servers to containerized +applications. + +#### Conclusion + +Ceph block devices represent a sophisticated implementation of block storage +that combines the traditional simplicity of block-based interfaces with modern +distributed storage capabilities. The thin-provisioned, resizable architecture +with data striping across multiple OSDs provides a foundation for scalable, +high-performance storage. Integration with RADOS brings enterprise features +including snapshotting, replication, and strong consistency, while support for +both kernel modules and librbd ensures broad compatibility across deployment +scenarios. The ability to run block devices alongside object and file storage +within a unified cluster positions Ceph as a comprehensive storage solution +capable of addressing diverse organizational storage requirements through a +single infrastructure platform. This convergence of capabilities, combined with +proven integration with major virtualization and cloud platforms, establishes +Ceph block devices as a viable solution for modern data center storage needs. + +### RADOS Gateway (RGW) in Summary + +#### RGW Overview + +RADOS Gateway, commonly referred to as RGW or radosgw, is Ceph's object storage +interface that provides applications with a RESTful gateway to store objects +and metadata in a Ceph cluster. As one of Ceph's three primary storage +interfaces alongside CephFS (file storage) and RBD (block storage), RGW +transforms Ceph's underlying RADOS object store into a scalable, S3 and +Swift-compatible object storage service. This enables organizations to build +cloud storage solutions that are compatible with industry-standard APIs while +leveraging Ceph's distributed architecture for reliability, scalability, and +performance. + +#### Architecture and Design + +RGW operates as an HTTP service that sits atop the Ceph +Storage Cluster. Unlike direct RADOS access, RGW provides a higher-level +abstraction specifically designed for object storage workloads. The gateway +maintains its own data formats, user database, authentication mechanisms, and +access control systems independent of the underlying Ceph cluster's +authentication. + +When a client stores data through RGW, the gateway receives HTTP requests, +authenticates the user, authorizes the operation, and then translates the +request into RADOS operations. Objects stored via RGW are ultimately persisted +as RADOS objects in the Ceph cluster, but RGW manages the mapping between +S3/Swift objects and the underlying RADOS objects. This abstraction layer allows +a single S3 or Swift object to potentially map to multiple RADOS objects, +particularly for large files that are striped across the cluster. + +#### API Compatibility + +One of RGW's most significant features is its dual API compatibility. RGW +provides RESTful interfaces compatible with both Amazon S3 and OpenStack Swift, +enabling applications designed for these platforms to work with Ceph without +modification. This compatibility extends beyond basic object operations to +include advanced features like multipart uploads, versioning, lifecycle +management, and bucket policies. + +The S3-compatible API supports a comprehensive set of operations including +bucket creation and deletion, object PUT/GET/DELETE operations, ACL management, +and metadata handling. The Swift-compatible API provides similar functionality +using Swift's terminology and conventions, with containers instead of buckets +and account/container/object hierarchy. Importantly, RGW implements a unified +namespace, meaning data written through the S3 API can be read through the Swift +API and vice versa, providing exceptional flexibility for multi-application +environments. + +#### Multi-Tenancy and User Management + +RGW implements sophisticated multi-tenancy capabilities that allow multiple +independent users and organizations to share the same Ceph cluster while +maintaining complete isolation. The system supports multiple authentication +mechanisms including built-in user management, LDAP integration, and integration +with external authentication systems like Keystone for OpenStack environments. + +Users in RGW are organized into a hierarchical structure. Each user belongs to a +tenant (which can be implicit or explicit), and users can have multiple access +keys for different applications or purposes. RGW manages user credentials, +quotas, and usage statistics independently, enabling service providers to offer +object storage as a multi-tenant service with per-user billing and resource +limits. + +#### Data Organization + +RGW organizes data using a bucket-based model for S3 compatibility (containers +in Swift terminology). Buckets are logical containers that hold objects, with +each bucket having its own policies, ACLs, and configuration. Objects within +buckets are identified by unique keys and can include arbitrary metadata +alongside the actual data payload. + +Internally, RGW uses multiple RADOS pools to organize different types of data. +Separate pools typically store bucket indexes, data objects, and metadata, +allowing administrators to apply different replication or erasure coding +strategies to different data types. For example, bucket indexes might use +replication for fast access while large data objects use erasure coding for +storage efficiency. + +#### Advanced Features + +RGW supports numerous advanced object storage features that make it suitable for +production deployments. Object versioning allows multiple versions of the same +object to coexist, enabling recovery from accidental overwrites or deletions. +Lifecycle management policies automate the transition of objects between storage +classes or deletion after specified periods, reducing storage costs and +administrative overhead. + +Server-side encryption provides data protection at rest, with support for +multiple encryption modes including customer-provided keys. Cross-origin +resource sharing (CORS) configuration enables web applications to access RGW +directly from browsers. Bucket notifications allow applications to receive +real-time events when objects are created, deleted, or modified, enabling +event-driven architectures. + +#### Scalability and Performance + +RGW's architecture enables horizontal scaling to meet growing storage and +throughput demands. Multiple RGW instances can be deployed behind load +balancers to distribute client requests across many gateways. Each RGW instance +operates independently, communicating directly with the underlying Ceph +cluster, avoiding any single point of contention. + +For improved performance, RGW implements various optimization strategies. It +can cache frequently accessed metadata to reduce request latency. Asynchronous +operations handle time-consuming tasks like +garbage collection and data synchronization without blocking client requests. +The gateway also supports byte-range requests, enabling efficient partial +object retrieval for large files and supporting features like HTTP video +streaming. + +#### Multi-Site Capabilities + +RGW includes robust multi-site replication capabilities for disaster recovery, +geographic distribution, and compliance requirements. The multi-site +architecture supports active-active configurations where multiple RGW clusters +can accept writes simultaneously, with changes automatically synchronized +across sites. This enables organizations to build globally distributed object +storage systems with local read/write access and automatic data replication. + +Metadata and data can be replicated independently with different strategies, +allowing for flexible topology designs. Zone groups organize multiple zones +(independent RGW deployments) into replication domains, while periods define +consistent configuration states across all zones. This sophisticated +replication framework supports complex scenarios like hub-and-spoke topologies, +full-mesh replication, and tiered storage architectures. + +#### Monitoring and Operations + +RGW provides comprehensive monitoring capabilities through usage statistics, +performance metrics, and administrative APIs. Administrators can track +bandwidth consumption, request rates, and storage utilization on a per-user or +per-bucket basis. Integration with standard monitoring tools allows RGW metrics +to be collected and visualized alongside other infrastructure components. + +The admin API enables programmatic management of users, buckets, and quotas, +facilitating automation and integration with billing systems or custom +management tools. Command-line tools provide capabilities for troubleshooting, +data inspection, and emergency operations. + +#### RGW Summary + +RADOS Gateway represents a mature, feature-rich object storage solution that +brings cloud-compatible APIs to Ceph's distributed storage platform. By +providing S3 and Swift compatibility, RGW enables organizations to build +private cloud storage solutions or offer object storage as a service while +maintaining control over their infrastructure. Its scalability, multi-tenancy +support, and advanced features make it suitable for use cases ranging from +backup and archive to content distribution and application data storage. As +part of the unified Ceph storage platform, RGW benefits from the same +reliability, performance, and operational characteristics that make Ceph a +leading choice for software-defined storage solutions. + +### CephFS in Summary + +#### CephFS Overview + +CephFS (Ceph File System) is Ceph's distributed file system interface that +provides POSIX-compliant file storage built on top of the RADOS object store. +As one of Ceph's three primary storage interfaces alongside RBD (block storage) +and RGW (object storage), CephFS enables users to mount a shared filesystem that +appears as a traditional hierarchical directory structure while leveraging +Ceph's distributed storage capabilities for scalability, reliability, and +performance. This combination of familiar filesystem semantics with enterprise +storage features makes CephFS suitable for workloads ranging from home +directories and shared application data to high-performance computing and big +data analytics. + +#### Architecture and Components + +CephFS operates through a carefully designed architecture that separates data +and metadata management. At its core, CephFS relies on two essential components: +the Metadata Server (MDS) and the underlying RADOS storage cluster that stores +both file data and metadata. + +The Metadata Server daemon (ceph-mds) manages all filesystem metadata including +directory structures, file ownership, permissions, access timestamps, and +extended attributes. Unlike traditional filesystems where metadata resides on +the same storage devices as data, CephFS stores metadata in dedicated RADOS +pools, allowing it to be replicated and distributed independently. This +separation enables CephFS to scale metadata operations independently of data +operations, a critical capability for large-scale deployments. + +File data in CephFS is stored as RADOS objects distributed across the cluster's +Object Storage Daemons (OSDs). When a client writes a file, CephFS stripes the +data across multiple objects according to configurable striping parameters, +enabling parallel I/O and leveraging the aggregate bandwidth of multiple storage +devices. This architecture allows CephFS to scale from gigabytes to petabytes +while maintaining consistent performance characteristics. + +#### POSIX Compliance and Compatibility + +CephFS provides strong POSIX compliance, supporting the vast majority of +standard filesystem operations expected by applications and users. This includes +hierarchical directory structures, standard file permissions and ownership, +symbolic and hard links, extended attributes, and file locking mechanisms. The +POSIX compliance allows many existing applications to use CephFS without +modification. Clients still require a CephFS kernel or FUSE mount, or a +`libcephfs` integration, together with the appropriate Ceph credentials. CephFS +is therefore not a protocol-level replacement for NFS or SMB. + +Clients can access CephFS through multiple methods. The kernel client integrates +directly with the Linux kernel, providing native filesystem performance and +supporting standard mount operations. FUSE (Filesystem in User Space) clients +enable CephFS mounting on systems without kernel module support or in situations +requiring non-root access. Additionally, libcephfs provides a library interface +for applications to interact with CephFS programmatically, enabling custom +integration scenarios. + +#### Metadata Server Design + +The MDS represents a sophisticated component designed specifically for +distributed metadata management. In CephFS, metadata operations like listing +directories, creating files, or checking permissions can dominate workload +patterns, particularly with applications handling many small files. By +maintaining metadata in memory and leveraging high-performance RADOS operations +for persistence, the MDS achieves low-latency metadata operations essential for +good filesystem performance. + +CephFS supports multiple MDS daemons operating simultaneously, enabling both +high availability and horizontal scalability. In active-passive configurations, +Ceph monitors assign a standby when an active MDS fails. Recovery time depends +on factors such as journal replay and whether standby-replay is configured. The +journal stored in RADOS provides the metadata needed for recovery. + +For scalability, CephFS implements dynamic subtree partitioning, allowing +multiple active MDS daemons to divide the filesystem namespace among themselves. +The system automatically balances load by migrating directory subtrees between +MDS instances based on access patterns. A heavily accessed directory can even be +sharded across multiple MDS daemons, with each daemon handling different entries +within the same directory. This dynamic load balancing ensures that metadata +operations scale with the number of active MDS instances. + +#### Performance Characteristics + +CephFS delivers strong performance across diverse workloads through several +architectural optimizations. Client-side caching reduces latency for frequently +accessed data and metadata, with cache coherency maintained through distributed +locking mechanisms managed by the MDS. This caching enables multiple clients to +access the same files efficiently while maintaining consistency. + +The striping of file data across multiple RADOS objects enables high-bandwidth +sequential I/O operations, with clients performing parallel reads and writes +directly to OSDs. For large files, this parallelism allows CephFS to saturate +available network bandwidth and leverage the aggregate throughput of many +storage devices simultaneously. + +Metadata performance benefits from the MDS's in-memory metadata cache and +efficient RADOS operations for persistence. For workloads with good locality, +where applications repeatedly access files within the same directory trees, the +MDS cache provides excellent performance. The ability to scale metadata +operations through multiple active MDS daemons addresses the metadata bottleneck +that plagues many distributed filesystems at scale. + +#### Snapshots and Quotas + +CephFS provides sophisticated snapshot capabilities enabling point-in-time +copies of directory trees. Snapshots are space-efficient, storing only changed +data rather than full copies, and can be created instantly on any directory +within the filesystem. Users can browse snapshot contents through a special +`.snap` directory and restore files or entire directory trees as needed. +Administrative snapshots enable backup and recovery strategies while +user-accessible snapshots provide self-service recovery from accidental +deletions or modifications. + +Directory quotas allow administrators to limit storage consumption at any point +in the directory hierarchy. Quotas can restrict both the total bytes consumed +and the number of files, with enforcement occurring at write time. This enables +multi-tenant deployments where different users or projects share a filesystem +while preventing any single entity from consuming excessive resources. + +#### Multiple Filesystems + +Recent CephFS versions support multiple independent filesystems within a single +Ceph cluster, each with its own namespace, MDS cluster, and data pools. This +capability enables isolation between different use cases or tenants while +sharing the underlying storage infrastructure. Each filesystem can be configured +with different parameters, replication strategies, or performance +characteristics appropriate to its specific workload requirements. + +#### Security and Access Control + +CephFS implements multiple layers of security. Path-based access restrictions +allow administrators to limit client access to specific directory subtrees, +enabling multi-tenant scenarios where different clients see only their allocated +portions of the filesystem. CephX authentication integrates with Ceph's native +authentication system, ensuring that only authorized clients can mount the +filesystem. + +Standard POSIX permissions and ACLs provide fine-grained access control at the +file and directory level, allowing familiar Unix-style permission management. +Extended attributes enable additional metadata storage for applications +requiring custom attributes or security labels. + +#### Use Cases and Applications + +CephFS excels in scenarios requiring shared filesystem access across multiple +clients. Home directories, shared application data, and collaborative workspaces +benefit from CephFS's strong consistency and POSIX compatibility. High +performance computing environments leverage CephFS for shared job data and +scratch space, taking advantage of the parallel I/O capabilities and scalability. + +Content creation workflows in media and entertainment utilize CephFS for shared +storage of large media files, benefiting from high bandwidth and the ability to +scale capacity and performance independently. Big data analytics platforms use +CephFS for storing datasets that multiple processing nodes must access +simultaneously. + +#### CephFS Summary + +CephFS represents a mature, scalable distributed filesystem that brings POSIX +compatibility to Ceph's distributed storage platform. By separating metadata and +data management, supporting multiple active MDS daemons, and leveraging RADOS +for reliable distributed storage, CephFS delivers enterprise-grade filesystem +capabilities suitable for demanding production workloads. Its combination of +familiar filesystem semantics, strong performance, and advanced features like +snapshots and dynamic metadata scaling makes CephFS a compelling choice for +organizations requiring shared filesystem storage at scale. + +## See Also -- [Storage - Rook](./rook) - Kubernetes operator that manages this Ceph cluster -- [Storage - Arbiter](./arbiter) - stretched cluster quorum -- [Ceph upstream architecture docs](https://docs.ceph.com/en/latest/architecture/) +The architecture of the Ceph cluster is explained in [the Architecture +chapter of the upstream Ceph +documentation](https://docs.ceph.com/en/latest/architecture/) diff --git a/docs/storage/chorus.md b/docs/storage/chorus.md index 964f52c..8f7b505 100644 --- a/docs/storage/chorus.md +++ b/docs/storage/chorus.md @@ -1,16 +1,32 @@ --- title: Chorus -order: 50 +order: 30 outline: deep --- # Chorus -::: tip Source Code -[github.com/cobaltcore-dev/chorus](https://github.com/cobaltcore-dev/chorus) +::: info +The Chorus source and detailed operational documentation currently require +CobaltCore organization access. ::: -Chorus provides zero-downtime data replication and migration between S3 and Swift object storage systems. It runs as a transparent proxy in front of a Ceph RGW cluster, enabling live migration from one object store to another without application downtime. +Chorus is data replication software designed for Object Storage systems, +supporting S3 and OpenStack Swift APIs. It enables zero-downtime migration +between storage systems, maintains synchronized backups for disaster recovery, +and verifies migration integrity through consistency checks. + +Chorus operates through two main components: Chorus Proxy, an S3 proxy that +captures changes, and Chorus Worker, which processes replication tasks and +webhook events. Users configure storage credentials, designating one endpoint +as "main" while others become "followers." Requests route through Chorus's S3 +API to the main storage and asynchronously replicate to follower endpoints. + +The system supports user-level and bucket-level replication policies, allowing +users to pause and resume replication via web admin UI or CLI. Chorus handles +initial replication of existing data in the background and can accept change +events via webhooks when proxy deployment isn't feasible, supporting S3 bucket +notifications and Swift access-log events. ## What Chorus does @@ -37,5 +53,4 @@ Replication policies are configured per user or per bucket. Webhooks are support ## See also -- [Chorus full documentation](https://github.com/cobaltcore-dev/chorus) -- [Storage - Ceph](./ceph) +- [Storage - Ceph](./ceph.md) diff --git a/docs/storage/index.md b/docs/storage/index.md index 6115c9a..87faa42 100644 --- a/docs/storage/index.md +++ b/docs/storage/index.md @@ -6,65 +6,73 @@ outline: deep # Storage -CobaltCore's storage layer is built on [Ceph](./ceph), a distributed storage system that delivers block, file, and object storage from a single unified cluster. The surrounding components handle Kubernetes-native lifecycle management, high-availability quorum in stretched clusters, object replication, and dynamic storage allocation. +CobaltCore's cloud storage layer is built on [Ceph](./ceph.md), a distributed storage system that delivers object, block, and file storage in a single unified platform. The surrounding components handle lifecycle automation, data replication, high-availability quorum, observability, and quota and usage metering - each with a focused responsibility. ## Architecture -The storage stack is organized into three layers: +The storage stack is organized into three layers, with observability providing +cross-cutting metrics, dashboards, alerting, and audit across them: **Foundation** - Ceph provides the core distributed storage engine. All other components either operate it, extend it, or observe it. -**Operations** - [Rook](./rook) manages the full Ceph cluster lifecycle as a Kubernetes operator. [Arbiter](./arbiter) extends quorum into stretched topologies by deploying external monitors at a tiebreaker site. +**Operations** - [Rook](./rook.md) runs as a Kubernetes operator and manages the full lifecycle of Ceph daemons (monitors, managers, OSDs, MDS, RGW) as containerized workloads. [Arbiter](./arbiter.md) extends quorum into stretched cluster topologies by deploying external Ceph monitors that Rook does not manage directly. -**Data Services** - [Chorus](./chorus) provides zero-downtime S3/Swift object replication and migration. [Liquid-Ceph](./liquid-ceph) enables dynamic, on-demand storage allocation. +**Data Services** - [Chorus](./chorus.md) provides zero-downtime data replication and migration between object storage systems (S3 and Swift). [Liquid-Ceph](./liquid-ceph.md) reports RGW quota, capacity, and usage data to Limes. ## Components | Component | Layer | Role | -|---|---|---| -| [Ceph](./ceph) | Foundation | Distributed storage engine - block (RBD), file (CephFS), object (RGW) | -| [Rook](./rook) | Operations | Kubernetes operator for Ceph lifecycle management | -| [Arbiter](./arbiter) | Operations | External Ceph monitors for quorum in stretched clusters | -| [Chorus](./chorus) | Data Services | Zero-downtime object storage replication and migration | -| [Liquid-Ceph](./liquid-ceph) | Data Services | Dynamic storage allocation across the Ceph cluster | - -## Storage interfaces - -| Interface | Use case | -|---|---| -| **RBD** (RADOS Block Device) | VM disks, database volumes - thin-provisioned, snapshotted block storage | -| **CephFS** | Shared POSIX filesystem for workloads that need file access across multiple VMs | -| **RGW** (RADOS Gateway) | S3 and Swift-compatible object storage for application data | - -## Data flow - -``` -Applications / VMs - │ -┌───────┴────────────────────┐ -│ RBD │ CephFS │ RGW │ ← Ceph interfaces -└───────┴────────────────────┘ - │ - RADOS (distributed object store) - │ - OSDs across cluster nodes - │ - ┌────┴─────┐ - │ Rook │ ← manages daemon lifecycle via Kubernetes CRDs - └──────────┘ - │ - ┌────┴──────┐ ┌─────────┐ ┌────────────┐ - │ Arbiter │ │ Chorus │ │ Liquid-Ceph│ - └───────────┘ └─────────┘ └────────────┘ - (quorum) (replication) (allocation) +|-----------|-------|------| +| [Ceph](./ceph.md) | Foundation | Distributed storage engine - block (RBD), file (CephFS), object (RGW) | +| [Rook](./rook.md) | Operations | Kubernetes operator for Ceph lifecycle management | +| [Arbiter](./arbiter.md) | Operations | External Ceph monitors for quorum in stretched clusters | +| [Chorus](./chorus.md) | Data Services | Zero-downtime object storage replication and migration | +| [Liquid-Ceph](./liquid-ceph.md) | Data Services | Limes integration for RGW quota, capacity, and usage metering | +| [Observability & Audit](./observability/) | Cross-cutting | Metrics, dashboards, alerting, and audit - Prometheus, Perses, Prysm | + +## Storage Interfaces + +Ceph exposes three storage interfaces that CobaltCore services consume: + +- **RBD (RADOS Block Device)** - thin-provisioned, resizable block volumes used by virtual machines and databases. Striped across OSDs for parallel I/O and backed by RADOS snapshots and replication. +- **CephFS** - POSIX-compliant distributed filesystem. Metadata is managed by a dedicated MDS cluster; data is striped across OSDs. Supports snapshots, quotas, and multiple active MDS daemons for horizontal metadata scaling. +- **RGW (RADOS Gateway)** - S3 and Swift-compatible object storage gateway. Supports multi-tenancy, versioning, lifecycle policies, server-side encryption, and multi-site active-active replication. + +## Component Relationships + +```mermaid +flowchart TB + Apps[Applications and VMs] + RBD[RBD] + CephFS[CephFS] + RGW[RGW] + RADOS[RADOS] + OSDs[OSDs across storage nodes] + MON[Ceph monitors] + + Apps --> RBD + Apps --> CephFS + Apps --> RGW + RBD --> RADOS + CephFS --> RADOS + RGW --> RADOS + RADOS --> OSDs + + Rook -. manages daemons .-> MON + Rook -. manages daemons .-> OSDs + Arbiter -. adds an external quorum member .-> MON + Chorus -. replicates objects .-> RGW + RGW -. exports quota, capacity, and usage .-> LiquidCeph[Liquid-Ceph] + LiquidCeph -. reports resources .-> Limes[Limes quota service] + Observability[Prometheus, Perses, and Prysm] -. monitors .-> RADOS ``` -## High availability +## High Availability -Ceph achieves HA through monitor quorum (3 or 5 monitors), OSD replication or erasure coding, and MDS standby daemons. In stretched deployments spanning two sites, [Arbiter](./arbiter) deploys a third monitor at a tiebreaker site to maintain quorum if one site goes offline. +Ceph achieves HA through monitor quorum (typically 3 or 5 monitors), OSD replication or erasure coding, and MDS standby daemons. In stretched deployments that span two sites, [Arbiter](./arbiter.md) deploys a third monitor at a tiebreaker site so that quorum is maintained even if one full site goes offline. -## See also +## See Also -- [Observability](/observability/) - Prometheus metrics, Perses dashboards, and Prysm for the storage stack -- [Ceph upstream architecture docs](https://docs.ceph.com/en/latest/architecture/) -- [Rook documentation](https://rook.io/docs/rook/latest-release/Getting-Started/intro/) +- [Ceph Upstream Architecture Docs](https://docs.ceph.com/en/latest/architecture/) +- [Rook Documentation](https://rook.io/docs/rook/latest-release/Getting-Started/intro/) +- [Observability & Audit](./observability/) - Prometheus metrics, Perses dashboards, and Prysm CLI for the storage stack diff --git a/docs/storage/liquid-ceph.md b/docs/storage/liquid-ceph.md index d78654f..fa949ef 100644 --- a/docs/storage/liquid-ceph.md +++ b/docs/storage/liquid-ceph.md @@ -5,22 +5,25 @@ order: 60 # Liquid-Ceph -::: tip Source Code -[github.com/cobaltcore-dev/liquid-ceph](https://github.com/cobaltcore-dev/liquid-ceph) -::: - -Liquid-Ceph enables dynamic, on-demand storage allocation across the Ceph cluster. It integrates with the [Liquid](https://github.com/sapcc/liquid) framework to expose Ceph storage capacity as a schedulable resource in the CobaltCore cloud. +Liquid-Ceph integrates Ceph RADOS Gateway (RGW) with Limes for quota and usage +metering. It reports object-storage capacity and consumption through the Liquid +service-provider API. ## What it does -Rather than statically pre-allocating Ceph pools and storage quotas, Liquid-Ceph allows capacity to be requested and released dynamically - storage is allocated when needed and returned to the shared pool when no longer required. This improves cluster utilization and simplifies capacity planning. +Liquid-Ceph maps RGW placement targets to Limes resources and availability +zones. Limes can then collect capacity and usage data and manage project quotas +for those resources. ## Integration -Liquid-Ceph implements the Liquid service provider interface, exposing Ceph RBD and RGW capacity to the Liquid scheduler. Nova and Cinder can use this to make storage placement decisions based on real-time availability. +Liquid-Ceph implements the Liquid service-provider interface for RGW. It uses +Keystone for authentication and reads Ceph and RGW information to expose quota, +capacity, and usage metrics to Limes. ::: info -Detailed deployment and configuration documentation is being expanded. See the [Liquid-Ceph GitHub repository](https://github.com/cobaltcore-dev/liquid-ceph) for current development state. +The Liquid-Ceph repository currently requires organization access. Public +deployment and configuration documentation is still being expanded. ::: ## See also diff --git a/docs/observability/index.md b/docs/storage/observability/index.md similarity index 96% rename from docs/observability/index.md rename to docs/storage/observability/index.md index 61e4159..e0d295e 100644 --- a/docs/observability/index.md +++ b/docs/storage/observability/index.md @@ -1,10 +1,10 @@ --- -title: Observability -order: 1 +title: Observability & Audit +order: 80 outline: deep --- -# Observability +# Observability & Audit CobaltCore's observability stack provides real-time visibility into every layer - compute, storage, networking, and OpenStack services. It combines Prometheus for metrics, Perses for dashboards, and Prysm for storage-specific monitoring and audit. diff --git a/docs/observability/perses.md b/docs/storage/observability/perses.md similarity index 100% rename from docs/observability/perses.md rename to docs/storage/observability/perses.md diff --git a/docs/observability/prometheus.md b/docs/storage/observability/prometheus.md similarity index 68% rename from docs/observability/prometheus.md rename to docs/storage/observability/prometheus.md index 53cbb03..5a7a281 100644 --- a/docs/observability/prometheus.md +++ b/docs/storage/observability/prometheus.md @@ -8,16 +8,20 @@ outline: deep Prometheus collects and stores time-series metrics from all CobaltCore components. It is the central metrics store that feeds alerting rules and Perses dashboards. -## Exporters +## Exporters and metric sources -| Exporter | Source | What it covers | +| Exporter or metric source | Source | What it covers | |---|---|---| | `ceph-exporter` | Ceph daemons | OSD stats, pool usage, cluster health, latency histograms | -| `rook-ceph-mgr` | Rook manager | Operator status, daemon lifecycle events | -| `radosgw-exporter` | RGW | Request rates, error rates, per-user and per-bucket bandwidth | -| `kvm-ha-agent` | Hypervisor nodes | Hypervisor uptime, VM instance counts, libvirt events | +| `ceph-mgr` metrics endpoint | Ceph Manager daemon | Cluster health, daemon status, and performance metrics | +| `prysm` | RGW and storage observability | Request rates, error rates, per-user and per-bucket bandwidth | +| `kvm-ha-agent` metrics | Hypervisor nodes | Hypervisor uptime, VM instance counts, libvirt events | | OpenStack exporters | Nova, Neutron, Cinder | API latency, queue depths, service health | +The entries above describe metric sources used by the stack; the exact exporters +and scrape targets depend on the deployment configuration. Prysm provides the +RADOS Gateway and storage-specific Prometheus metrics. + ## Retention and storage Metrics are retained according to the cluster-wide retention policy. Long-term storage uses Prometheus remote-write to an external TSDB (configured separately per deployment). diff --git a/docs/observability/prysm.md b/docs/storage/observability/prysm.md similarity index 98% rename from docs/observability/prysm.md rename to docs/storage/observability/prysm.md index eb59cde..22ccfb4 100644 --- a/docs/observability/prysm.md +++ b/docs/storage/observability/prysm.md @@ -63,4 +63,4 @@ Prysm supports three output modes: ## See also - [Observability - Prometheus](./prometheus) -- [Storage - Ceph](../storage/ceph) +- [Storage - Ceph](../ceph) diff --git a/docs/storage/rook-ceph.md b/docs/storage/rook-ceph.md new file mode 100644 index 0000000..d5f6091 --- /dev/null +++ b/docs/storage/rook-ceph.md @@ -0,0 +1,398 @@ +--- +title: Installing Rook-Ceph on Kubernetes +order: 25 +--- + +# Installing Rook-Ceph on Kubernetes + +## Overview + +This guide provides step-by-step instructions for deploying a Ceph storage +cluster using the Rook operator on Kubernetes. Rook automates the deployment, +configuration, and management of Ceph clusters within Kubernetes environments. + +The instructions here are meant only as a general guideline. We recommend that +you use the instructions found in the [official Rook +documentation](https://rook.io/docs/rook/latest/) and the [upstream Ceph +documentation](https://docs.ceph.com/). + +## Prerequisites + +Before beginning the installation, ensure the following requirements are met: + +### Kubernetes Cluster Requirements + +- Kubernetes v1.31 through v1.37 +- `kubectl` configured to communicate with your cluster +- Administrator access to the Kubernetes cluster +- At least 3 worker nodes for the standard `cluster.yaml` used by this guide +- Verify compatibility between your Kubernetes version and the Rook version you + intend to deploy - see the [Rook releases page](https://github.com/rook/rook/releases) + for version compatibility information + +### Storage Requirements + +- Raw block devices available on worker nodes (unformatted, no filesystem) +- At least 10 GB per OSD for testing; size production OSDs for the expected + workload, recovery headroom, and Ceph release guidance +- Devices should not be mounted or in use by the operating system + +### Network Requirements + +- Network connectivity between all cluster nodes +- Network access between pods is handled by the Kubernetes network plugin (CNI). + Ensure your CNI supports the required pod-to-pod communication. If you need + to open ports for external access to Ceph services, the typical ports are + 6789, 3300, and 6800-7300. + +### System Requirements + +- A Linux kernel supported by the selected Kubernetes and Ceph releases +- LVM2 packages installed on all nodes +- CPU and memory sized for the planned Ceph daemons and workload +- `helm` installed if using Helm-based deployment (optional) + +## Install Rook-Ceph + +The examples below pin Rook `v1.20.7`, which supports the Kubernetes range +listed above. Select another tagged release and follow its compatibility guide +if your Kubernetes version falls outside that range. + +```bash +git clone --depth 1 --branch v1.20.7 https://github.com/rook/rook.git +cd rook/deploy/examples + +# Install the Rook operator +kubectl create -f crds.yaml +kubectl create -f common.yaml +kubectl create -f csi-operator.yaml +kubectl create -f operator.yaml + +# Wait for the operator before configuring the Ceph cluster +kubectl -n rook-ceph rollout status deployment/rook-ceph-operator + +# Install the CSI driver resources reconciled by the CSI operator +helm repo add ceph-csi-operator https://ceph.github.io/ceph-csi-operator +helm repo update +helm install ceph-csi-drivers ceph-csi-operator/ceph-csi-drivers \ + --namespace rook-ceph \ + --version 1.0.4 \ + --wait \ + -f https://raw.githubusercontent.com/rook/rook/v1.20.7/deploy/charts/ceph-csi-drivers/values.yaml +``` + +Run the remaining commands from the `rook/deploy/examples` directory so that +paths such as `cluster.yaml`, `filesystem.yaml`, and `csi/` resolve correctly. + +## Configuration Options + +### Customizing the Cluster + +Edit `cluster.yaml` to customize your deployment before creating the cluster: + +#### Storage Configuration + +Specify which devices to use for OSDs: + +```yaml +storage: + useAllNodes: true + useAllDevices: false + deviceFilter: "^sd[b-z]" # Use sdb, sdc, etc. +``` + +Or specify devices explicitly: + +```yaml +storage: + useAllNodes: false + useAllDevices: false + nodes: + - name: "node1" + devices: + - name: "sdb" + - name: "node2" + devices: + - name: "sdc" + - name: "node3" + devices: + - name: "sdd" +``` + +#### Resource Limits + +Set resource limits for Ceph daemons: + +```yaml +resources: + mon: + limits: + cpu: "2000m" + memory: "4Gi" + requests: + cpu: "1000m" + memory: "2Gi" + osd: + limits: + cpu: "2000m" + memory: "4Gi" + requests: + cpu: "1000m" + memory: "2Gi" +``` + +#### Network Configuration + +Configure network settings for client and cluster traffic: + +```yaml +network: + provider: host # or multus for advanced networking + # Uncomment for dual network configuration + # connections: + # encryption: + # enabled: true +``` + +## Deploy the Ceph Cluster + +After reviewing the settings above and updating `cluster.yaml`, create the +cluster and wait for it to become ready: + +::: danger Select storage devices before deployment +The upstream `cluster.yaml` enables `useAllDevices: true`. Do not apply it until +you have set `useAllDevices: false` and selected the intended devices, unless +every eligible raw device on every selected node is dedicated to Ceph. +::: + +```bash +kubectl create -f cluster.yaml +kubectl -n rook-ceph get cephcluster -w +``` + +Press `Ctrl+C` after the cluster reports `Ready`. + +### Dashboard Access + +Enable and access the Ceph dashboard: + +```bash +# The dashboard is enabled by default in cluster.yaml + +# Get the dashboard password +kubectl -n rook-ceph get secret rook-ceph-dashboard-password \ + -o jsonpath="{.data.password}" | base64 --decode && echo + +# Port-forward to access the dashboard +kubectl -n rook-ceph port-forward service/rook-ceph-mgr-dashboard 8443:8443 +``` + +Access the dashboard at: `https://localhost:8443` + +Username: `admin` +Password: (from the command above) + +## Creating Storage Classes + +### Block Storage (RBD) + +Create a storage class for block devices: + +```bash +kubectl create -f csi/rbd/storageclass.yaml +``` + +### File Storage (CephFS) + +Deploy the CephFS filesystem: + +```bash +kubectl create -f filesystem.yaml +``` + +Create a storage class for shared filesystem: + +```bash +kubectl create -f csi/cephfs/storageclass.yaml +``` + +### Object Storage (RGW) + +Deploy the object storage service: + +```bash +kubectl create -f object.yaml +``` + +Wait for the RGW pods to be ready: + +```bash +kubectl -n rook-ceph wait --for=condition=Ready pod \ + -l app=rook-ceph-rgw --timeout=5m +``` + +## Verification + +### Install the Ceph Toolbox + +Install the toolbox used by the health and troubleshooting commands below: + +```bash +kubectl create -f toolbox.yaml +kubectl -n rook-ceph rollout status deployment/rook-ceph-tools +``` + +### Verify All Storage Types + +Check that all storage components are operational: + +```bash +# Check block storage +kubectl get storageclass rook-ceph-block + +# Check filesystem storage +kubectl get storageclass rook-cephfs + +# Check object storage +kubectl -n rook-ceph get cephobjectstore +``` + +### Test Storage Functionality + +Create test workloads using each storage type: + +```bash +# Test RBD block storage +kubectl create -f csi/rbd/pvc.yaml +kubectl create -f csi/rbd/pod.yaml +kubectl wait --for=condition=Ready pod/csirbd-demo-pod --timeout=5m + +# Test CephFS +kubectl create -f csi/cephfs/pvc.yaml +kubectl create -f csi/cephfs/pod.yaml +kubectl wait --for=condition=Ready pod/csicephfs-demo-pod --timeout=5m +``` + +## Troubleshooting + +### Common Issues + +**Operator not starting:** + +```bash +# Check operator logs +kubectl -n rook-ceph logs -l app=rook-ceph-operator +``` + +**OSDs not starting:** + +```bash +# Check OSD prepare logs +kubectl -n rook-ceph logs -l app=rook-ceph-osd-prepare +``` + +Run these read-only checks directly on the affected storage node, replacing +`/dev/sdX` with the intended OSD device: + +```bash +lsblk -f +sudo wipefs --no-act /dev/sdX +``` + +**Cluster stuck in HEALTH_WARN:** + +```bash +# Check detailed cluster status +kubectl -n rook-ceph exec -it deployment/rook-ceph-tools -- ceph health detail + +# Check for common issues +kubectl -n rook-ceph exec -it deployment/rook-ceph-tools -- ceph -s +``` + +## Cleanup + +To remove the Rook-Ceph cluster: + +**Note:** Rook uses Kubernetes finalizers to protect resources from accidental +deletion and to enforce dependency order. Do not remove those finalizers +manually. If deletion stalls, check for dependent resources and operator errors +before continuing. See the +[Rook cleanup documentation](https://rook.io/docs/rook/latest/Storage-Configuration/ceph-teardown/) +for details. + +```bash +# Delete the example workloads and claims created by this guide +kubectl delete -f csi/rbd/pod.yaml --ignore-not-found +kubectl delete -f csi/rbd/pvc.yaml --ignore-not-found +kubectl delete -f csi/cephfs/pod.yaml --ignore-not-found +kubectl delete -f csi/cephfs/pvc.yaml --ignore-not-found + +# Delete the storage classes +kubectl delete -f csi/rbd/storageclass.yaml --ignore-not-found +kubectl delete -f csi/cephfs/storageclass.yaml --ignore-not-found + +# Delete the troubleshooting toolbox +kubectl delete -f toolbox.yaml --ignore-not-found + +# Delete object storage (if created) +kubectl delete -f object.yaml --ignore-not-found + +# Delete filesystem (if created) +kubectl delete -f filesystem.yaml --ignore-not-found + +# Delete the cluster after its dependent resources are gone +kubectl delete -f cluster.yaml +kubectl -n rook-ceph wait --for=delete cephcluster/rook-ceph --timeout=15m + +# Delete the CSI driver chart +helm uninstall --namespace rook-ceph ceph-csi-drivers + +# Delete the operator +kubectl delete -f operator.yaml +kubectl delete -f csi-operator.yaml +kubectl delete -f common.yaml +kubectl delete -f crds.yaml +``` + +**Cleaning up storage on nodes (CAUTION: This deletes all data):** + +Run the following on each node that had OSDs. In addition to removing the Rook +data directory, the raw block devices used by OSDs must be wiped before they +can be reused: + +```bash +# Remove Rook data directory +sudo rm -rf /var/lib/rook + +# Wipe each OSD device (replace /dev/sdX with the actual device name) +sudo sgdisk --zap-all /dev/sdX +sudo wipefs --all /dev/sdX +sudo partprobe /dev/sdX +``` + +## Next Steps + +After successful installation: + +1. Configure monitoring with Prometheus and Grafana +2. Set up backup and disaster recovery procedures +3. Implement resource quotas and limits +4. Configure advanced networking if required +5. Review and adjust Ceph configuration parameters +6. Set up regular maintenance schedules + +## Additional Resources + +- [Official Rook documentation](https://rook.io/docs/rook/latest/) +- [Ceph documentation](https://docs.ceph.com/) +- [Rook GitHub repository](https://github.com/rook/rook) +- [Rook Slack community](https://slack.rook.io/) + +## Notes + +- This guide provides a basic Rook-Ceph deployment. While the prerequisites + describe a production-grade setup, additional considerations apply for + production environments, including high availability, performance tuning, + and security hardening. +- Always test deployment procedures in a non-production environment first. +- Keep Rook and Ceph versions updated for security and stability improvements. diff --git a/docs/storage/rook.md b/docs/storage/rook.md index 1dd33c9..5895b21 100644 --- a/docs/storage/rook.md +++ b/docs/storage/rook.md @@ -10,11 +10,45 @@ outline: deep [github.com/rook/rook](https://github.com/rook/rook) ::: -Rook is the Kubernetes operator that manages the Ceph cluster in CobaltCore. It runs Ceph daemons (monitors, managers, OSDs, MDS, RGW) as containerized Kubernetes workloads and provides declarative management through CRDs. +Rook is an open-source cloud-native storage orchestrator that automates the +deployment, configuration, and management of [Ceph](./ceph.md) storage clusters +within Kubernetes environments. Built as a Kubernetes operator, Rook extends +Kubernetes with custom resource definitions (CRDs) that allow administrators to +define and manage Ceph clusters using native Kubernetes APIs and tools. + +Rook eliminates much of the operational complexity traditionally associated +with running Ceph by leveraging Kubernetes primitives for scheduling, +self-healing, and scaling. When deployed, Rook runs as a set of pods within the +Kubernetes cluster, managing the lifecycle of Ceph daemons (monitors, managers, +OSDs, MDS, and RGW) as containerized workloads. It automatically handles tasks +such as OSD provisioning from available storage devices and managing the monitor +quorum. + +The system provides declarative configuration through YAML manifests, enabling +infrastructure-as-code practices for storage management. Administrators can +define storage classes that map to Ceph pools, allowing applications to +dynamically provision persistent volumes for block storage (RBD) and shared +file systems (CephFS) through standard Kubernetes mechanisms. Object storage +(RGW) is exposed separately through a `CephObjectStore` resource and S3 or +Swift-compatible APIs. + +Rook continuously monitors cluster health and automatically responds to +failures by restarting failed daemon pods and maintaining the desired state +defined in the cluster specifications. Replacing a failed storage device still +requires an administrator to prepare the replacement and remove the failed OSD. +It integrates with [Kubernetes](https://kubernetes.io/) monitoring and logging systems, +providing visibility into storage operations alongside application workloads. ## Why Rook? -Running Ceph as a Kubernetes workload means the cluster lifecycle - initial deployment, scaling, upgrades, and self-healing - is handled by Kubernetes controllers rather than manual playbooks. Rook bridges the gap between Ceph's daemon model and Kubernetes' declarative model. +Running Ceph as a Kubernetes workload means the cluster lifecycle - initial +deployment, scaling, upgrades, and self-healing - is handled by Kubernetes +controllers rather than manual playbooks. Rook bridges the gap between Ceph's +daemon model and Kubernetes' declarative model. + +CobaltCore's cloud infrastructure and automation foundation are built on +Kubernetes. Rook is therefore used to manage Ceph workloads through the same +declarative control plane. ## How Rook manages Ceph @@ -34,10 +68,13 @@ graph LR ### Prerequisites -- Kubernetes 1.25+ +- Kubernetes 1.31 through 1.37 - Raw block devices available on storage nodes (unformatted, no filesystem) - Network connectivity between storage nodes +The commands below pin Rook `v1.20.7` and Ceph `v20.2.4` so the chart, +manifests, and compatibility range remain consistent. + ### Install the operator ```bash @@ -46,13 +83,34 @@ helm repo update helm install --create-namespace \ --namespace rook-ceph \ + --version v1.20.7 \ + --wait \ rook-ceph rook-release/rook-ceph + +helm repo add ceph-csi-operator https://ceph.github.io/ceph-csi-operator +helm repo update + +helm install --namespace rook-ceph \ + --version 1.0.4 \ + --wait \ + -f https://raw.githubusercontent.com/rook/rook/v1.20.7/deploy/charts/ceph-csi-drivers/values.yaml \ + ceph-csi-drivers ceph-csi-operator/ceph-csi-drivers ``` +The `rook-ceph` chart installs the Ceph-CSI operator and its CRDs. The second +chart installs the driver resources reconciled by that operator. + ### Deploy the Ceph cluster Create a `CephCluster` resource. A minimal 3-node cluster: +::: danger Dedicated devices only +The example sets `useAllDevices: true`. Rook will consume every eligible raw +device it discovers on the selected nodes. Use dedicated storage nodes, or set +both `useAllNodes` and `useAllDevices` to `false` and select devices explicitly +before applying the manifest. +::: + ```yaml apiVersion: ceph.rook.io/v1 kind: CephCluster @@ -61,7 +119,7 @@ metadata: namespace: rook-ceph spec: cephVersion: - image: quay.io/ceph/ceph:v18 + image: quay.io/ceph/ceph:v20.2.4 dataDirHostPath: /var/lib/rook mon: count: 3 @@ -83,23 +141,24 @@ kubectl get cephcluster -n rook-ceph -w ### Create storage classes **RBD (block):** + ```bash -kubectl apply -f https://raw.githubusercontent.com/rook/rook/refs/heads/release-1.17/deploy/examples/csi/rbd/storageclass.yaml +kubectl apply -f https://raw.githubusercontent.com/rook/rook/v1.20.7/deploy/examples/csi/rbd/storageclass.yaml ``` **CephFS (file):** + ```bash -kubectl apply -f https://raw.githubusercontent.com/rook/rook/refs/heads/release-1.17/deploy/examples/csi/cephfs/storageclass.yaml +kubectl apply -f https://raw.githubusercontent.com/rook/rook/v1.20.7/deploy/examples/filesystem.yaml +kubectl apply -f https://raw.githubusercontent.com/rook/rook/v1.20.7/deploy/examples/csi/cephfs/storageclass.yaml ``` -Replace `release-1.17` with the Rook version you installed. - ### Verify First install the Rook toolbox to get access to `ceph` CLI commands: ```bash -kubectl apply -f https://raw.githubusercontent.com/rook/rook/refs/heads/release-1.17/deploy/examples/toolbox.yaml +kubectl apply -f https://raw.githubusercontent.com/rook/rook/v1.20.7/deploy/examples/toolbox.yaml kubectl rollout status deployment/rook-ceph-tools -n rook-ceph ```