From 8bae33f4f9f7ba9548953360a8ccb3e6ca3d8070 Mon Sep 17 00:00:00 2001
From: Austin Barrington <austin@hyperbyte.dev>
Date: Fri, 4 Sep 2026 18:00:54 +0100
Subject: [PATCH] fix(backup): snapshot with hyperbytedb and upload with
 minio/mc

The database image has no AWS CLI, so Jobs that ran aws s3 sync always failed.
Use an mc sidecar against S3-compatible storage (MinIO locally) and keep restore
Job names under 63 bytes so Kubernetes will accept them.
---
 internal/controller/backup_job_test.go        | 144 +++++++++++++++
 .../hyperbytedbbackup_controller.go           | 113 +-----------
 .../hyperbytedbrestore_controller.go          |  86 +--------
 internal/controller/s3_transfer.go            | 171 ++++++++++++++++++
 4 files changed, 327 insertions(+), 187 deletions(-)
 create mode 100644 internal/controller/backup_job_test.go
 create mode 100644 internal/controller/s3_transfer.go

diff --git a/internal/controller/backup_job_test.go b/internal/controller/backup_job_test.go
new file mode 100644
index 0000000..9faa97f
--- /dev/null
+++ b/internal/controller/backup_job_test.go
@@ -0,0 +1,144 @@
+package controller
+
+import (
+	"strings"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
+
+	hyperbytedbv1alpha1 "github.com/hyperbyte-cloud/hyperbytedb-operator/api/v1alpha1"
+)
+
+func testBackupCluster() *hyperbytedbv1alpha1.HyperbytedbCluster {
+	return &hyperbytedbv1alpha1.HyperbytedbCluster{
+		ObjectMeta: metav1.ObjectMeta{Name: "db-test", Namespace: "tenant-1"},
+		Spec: hyperbytedbv1alpha1.HyperbytedbClusterSpec{
+			Replicas: ptr.To(int32(1)),
+			Image:    "ghcr.io/hyperbyte-cloud/hyperbytedb:0.8.5-beta",
+		},
+	}
+}
+
+func testBackup() *hyperbytedbv1alpha1.HyperbytedbBackup {
+	return &hyperbytedbv1alpha1.HyperbytedbBackup{
+		ObjectMeta: metav1.ObjectMeta{Name: "db-test-backup-abcd", Namespace: "tenant-1"},
+		Spec: hyperbytedbv1alpha1.HyperbytedbBackupSpec{
+			ClusterName: "db-test",
+			Destination: hyperbytedbv1alpha1.BackupDestination{
+				S3: hyperbytedbv1alpha1.S3BackupSpec{
+					Bucket:                "hyperbytedb-backups",
+					Prefix:                "inst-1",
+					Region:                "us-east-1",
+					Endpoint:              "http://minio.platform.svc.cluster.local:9000",
+					CredentialsSecretName: "hyperbytedb-backup-s3",
+				},
+			},
+			RetentionDays: 7,
+		},
+	}
+}
+
+func TestBuildBackupJobSnapshotsThenUploadsWithMc(t *testing.T) {
+	r := &HyperbytedbBackupReconciler{}
+	job := r.buildBackupJob(testBackup(), testBackupCluster(), "db-test-backup-abcd")
+	spec := job.Spec.Template.Spec
+
+	if len(spec.InitContainers) != 1 {
+		t.Fatalf("init containers = %d, want 1 (hyperbytedb snapshot)", len(spec.InitContainers))
+	}
+	init := spec.InitContainers[0]
+	if init.Name != "snapshot" {
+		t.Errorf("init name = %q, want snapshot", init.Name)
+	}
+	if init.Image != "ghcr.io/hyperbyte-cloud/hyperbytedb:0.8.5-beta" {
+		t.Errorf("snapshot image = %q", init.Image)
+	}
+	script := strings.Join(init.Command, " ")
+	if !strings.Contains(script, "hyperbytedb backup") {
+		t.Errorf("snapshot command missing hyperbytedb backup: %s", script)
+	}
+	if strings.Contains(script, "aws ") {
+		t.Errorf("snapshot must not call aws: %s", script)
+	}
+
+	if len(spec.Containers) != 1 {
+		t.Fatalf("containers = %d, want 1 (mc upload)", len(spec.Containers))
+	}
+	up := spec.Containers[0]
+	if up.Name != "upload" {
+		t.Errorf("container name = %q, want upload", up.Name)
+	}
+	if !strings.Contains(up.Image, "minio/mc") {
+		t.Errorf("upload image = %q, want minio/mc", up.Image)
+	}
+	upScript := strings.Join(up.Command, " ")
+	if !strings.Contains(upScript, "mc ") {
+		t.Errorf("upload command missing mc: %s", upScript)
+	}
+	if strings.Contains(upScript, "aws ") {
+		t.Errorf("upload must not call aws: %s", upScript)
+	}
+
+	names := map[string]bool{}
+	for _, v := range spec.Volumes {
+		names[v.Name] = true
+	}
+	if !names["snapshot"] || !names["data"] {
+		t.Errorf("volumes = %v, want snapshot emptyDir and data PVC", names)
+	}
+}
+
+func TestBuildRestoreJobDownloadsWithMcThenRestores(t *testing.T) {
+	r := &HyperbytedbRestoreReconciler{}
+	restore := &hyperbytedbv1alpha1.HyperbytedbRestore{
+		ObjectMeta: metav1.ObjectMeta{Name: "db-test-restore-abcd", Namespace: "tenant-1"},
+		Spec: hyperbytedbv1alpha1.HyperbytedbRestoreSpec{
+			ClusterName: "db-test",
+			BackupName:  "db-test-backup-abcd",
+		},
+	}
+	s3 := &hyperbytedbv1alpha1.S3BackupSpec{
+		Bucket:                "hyperbytedb-backups",
+		Prefix:                "inst-1",
+		Endpoint:              "http://minio.platform.svc.cluster.local:9000",
+		CredentialsSecretName: "hyperbytedb-backup-s3",
+	}
+	job := r.buildRestoreJob(restore, testBackupCluster(), s3, 0)
+	spec := job.Spec.Template.Spec
+
+	if len(spec.InitContainers) != 1 {
+		t.Fatalf("init containers = %d, want 1 (mc download)", len(spec.InitContainers))
+	}
+	dl := spec.InitContainers[0]
+	if dl.Name != "download" {
+		t.Errorf("init name = %q, want download", dl.Name)
+	}
+	if !strings.Contains(dl.Image, "minio/mc") {
+		t.Errorf("download image = %q, want minio/mc", dl.Image)
+	}
+	if strings.Contains(strings.Join(dl.Command, " "), "aws ") {
+		t.Errorf("download must not call aws: %s", dl.Command)
+	}
+
+	if len(spec.Containers) != 1 {
+		t.Fatalf("containers = %d, want 1 (hyperbytedb restore)", len(spec.Containers))
+	}
+	rst := spec.Containers[0]
+	if rst.Image != "ghcr.io/hyperbyte-cloud/hyperbytedb:0.8.5-beta" {
+		t.Errorf("restore image = %q", rst.Image)
+	}
+	script := strings.Join(rst.Command, " ")
+	if !strings.Contains(script, "hyperbytedb restore") {
+		t.Errorf("restore command missing hyperbytedb restore: %s", script)
+	}
+	if strings.Contains(script, "aws ") {
+		t.Errorf("restore must not call aws: %s", script)
+	}
+
+	restore.Name = "db-7f4eaf37-8805-46e4-97d2-6a4c7d18c0e2-restore-3f839e85"
+	job = r.buildRestoreJob(restore, testBackupCluster(), s3, 0)
+	if n := len(job.Name); n > 63 {
+		t.Fatalf("restore job name %q is %d bytes; Kubernetes job-name labels must be <=63", job.Name, n)
+	}
+}
diff --git a/internal/controller/hyperbytedbbackup_controller.go b/internal/controller/hyperbytedbbackup_controller.go
index f8d0a37..862ea0d 100644
--- a/internal/controller/hyperbytedbbackup_controller.go
+++ b/internal/controller/hyperbytedbbackup_controller.go
@@ -207,94 +207,7 @@ func (r *HyperbytedbBackupReconciler) reconcileCronJob(ctx context.Context, back
 }
 
 func (r *HyperbytedbBackupReconciler) buildBackupJob(backup *hyperbytedbv1alpha1.HyperbytedbBackup, cluster *hyperbytedbv1alpha1.HyperbytedbCluster, name string) *batchv1.Job {
-	image := hyperbytedb.ResolveHyperbytedbImage(cluster)
-
 	s3 := backup.Spec.Destination.S3
-	s3Path := r.buildBackupS3Path(backup)
-
-	backupScript := fmt.Sprintf(`#!/bin/sh
-set -e
-BACKUP_DIR="/tmp/backup"
-TIMESTAMP=$(date +%%Y%%m%%d-%%H%%M%%S)
-S3_DEST="s3://%s/${TIMESTAMP}/"
-
-echo "Starting hyperbytedb backup..."
-hyperbytedb backup --output "${BACKUP_DIR}"
-
-BACKUP_SIZE=$(du -sh "${BACKUP_DIR}" | cut -f1)
-echo "Backup size: ${BACKUP_SIZE}"
-
-echo "Uploading to ${S3_DEST}..."
-`, s3Path)
-
-	if s3.Endpoint != "" {
-		backupScript += fmt.Sprintf(`aws s3 sync "${BACKUP_DIR}" "${S3_DEST}" --endpoint-url "%s"
-`, s3.Endpoint)
-	} else {
-		backupScript += `aws s3 sync "${BACKUP_DIR}" "${S3_DEST}"
-`
-	}
-
-	backupScript += fmt.Sprintf(`
-echo "Cleaning up backups older than %d days..."
-`, backup.Spec.RetentionDays)
-
-	if s3.Endpoint != "" {
-		backupScript += fmt.Sprintf(`CUTOFF=$(date -d "-%d days" +%%Y%%m%%d-000000 2>/dev/null || date -v -%dd +%%Y%%m%%d-000000 2>/dev/null || echo "")
-if [ -n "$CUTOFF" ]; then
-  aws s3 ls "s3://%s/" --endpoint-url "%s" | while read -r line; do
-    DIR=$(echo "$line" | awk '{print $NF}' | tr -d '/')
-    if [ "$DIR" \< "$CUTOFF" ] 2>/dev/null; then
-      echo "Removing old backup: ${DIR}"
-      aws s3 rm "s3://%s/${DIR}/" --recursive --endpoint-url "%s"
-    fi
-  done
-fi
-`, backup.Spec.RetentionDays, backup.Spec.RetentionDays, s3Path, s3.Endpoint, s3Path, s3.Endpoint)
-	} else {
-		backupScript += fmt.Sprintf(`CUTOFF=$(date -d "-%d days" +%%Y%%m%%d-000000 2>/dev/null || date -v -%dd +%%Y%%m%%d-000000 2>/dev/null || echo "")
-if [ -n "$CUTOFF" ]; then
-  aws s3 ls "s3://%s/" | while read -r line; do
-    DIR=$(echo "$line" | awk '{print $NF}' | tr -d '/')
-    if [ "$DIR" \< "$CUTOFF" ] 2>/dev/null; then
-      echo "Removing old backup: ${DIR}"
-      aws s3 rm "s3://%s/${DIR}/" --recursive
-    fi
-  done
-fi
-`, backup.Spec.RetentionDays, backup.Spec.RetentionDays, s3Path, s3Path)
-	}
-
-	backupScript += `echo "Backup complete"
-`
-
-	env := []corev1.EnvVar{}
-	if s3.CredentialsSecretName != "" {
-		env = append(env,
-			corev1.EnvVar{
-				Name: "AWS_ACCESS_KEY_ID",
-				ValueFrom: &corev1.EnvVarSource{
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{Name: s3.CredentialsSecretName},
-						Key:                  "access_key_id",
-					},
-				},
-			},
-			corev1.EnvVar{
-				Name: "AWS_SECRET_ACCESS_KEY",
-				ValueFrom: &corev1.EnvVarSource{
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{Name: s3.CredentialsSecretName},
-						Key:                  "secret_access_key",
-					},
-				},
-			},
-		)
-	}
-	if s3.Region != "" {
-		env = append(env, corev1.EnvVar{Name: "AWS_DEFAULT_REGION", Value: s3.Region})
-	}
-
 	return &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
@@ -305,28 +218,12 @@ fi
 			BackoffLimit: ptr.To(int32(3)),
 			Template: corev1.PodTemplateSpec{
 				Spec: corev1.PodSpec{
-					RestartPolicy: corev1.RestartPolicyOnFailure,
-					Containers: []corev1.Container{
-						{
-							Name:    "backup",
-							Image:   image,
-							Command: []string{"sh", "-c", backupScript},
-							Env:     env,
-							VolumeMounts: []corev1.VolumeMount{
-								{Name: "data", MountPath: "/var/lib/hyperbytedb", ReadOnly: true},
-							},
-						},
-					},
+					RestartPolicy:  corev1.RestartPolicyOnFailure,
+					InitContainers: []corev1.Container{snapshotBackupInitContainer(hyperbytedb.ResolveHyperbytedbImage(cluster))},
+					Containers:     []corev1.Container{mcUploadContainer(s3, backup.Spec.RetentionDays)},
 					Volumes: []corev1.Volume{
-						{
-							Name: "data",
-							VolumeSource: corev1.VolumeSource{
-								PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
-									ClaimName: fmt.Sprintf("data-%s-0", cluster.Name),
-									ReadOnly:  true,
-								},
-							},
-						},
+						dataVolume(cluster.Name, 0, true),
+						snapshotVolume(),
 					},
 				},
 			},
diff --git a/internal/controller/hyperbytedbrestore_controller.go b/internal/controller/hyperbytedbrestore_controller.go
index 7b84121..4576c29 100644
--- a/internal/controller/hyperbytedbrestore_controller.go
+++ b/internal/controller/hyperbytedbrestore_controller.go
@@ -19,7 +19,6 @@ package controller
 import (
 	"context"
 	"fmt"
-	"strings"
 	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -187,7 +186,7 @@ func (r *HyperbytedbRestoreReconciler) checkRestoreJob(ctx context.Context, rest
 	anyFailed := false
 
 	for i := int32(0); i < replicas; i++ {
-		jobName := fmt.Sprintf("%s-restore-%d", restore.Name, i)
+		jobName := fmt.Sprintf("%s-%d", restore.Name, i)
 		job := &batchv1.Job{}
 		if err := r.Get(ctx, types.NamespacedName{Name: jobName, Namespace: restore.Namespace}, job); err != nil {
 			if apierrors.IsNotFound(err) {
@@ -246,65 +245,9 @@ func (r *HyperbytedbRestoreReconciler) buildRestoreJob(
 	s3Source *hyperbytedbv1alpha1.S3BackupSpec,
 	ordinal int32,
 ) *batchv1.Job {
-	image := hyperbytedb.ResolveHyperbytedbImage(cluster)
-
-	s3Path := s3Source.Bucket
-	if s3Source.Prefix != "" {
-		s3Path += "/" + strings.TrimSuffix(s3Source.Prefix, "/")
-	}
-
-	syncCmd := fmt.Sprintf(`aws s3 sync "s3://%s/" /tmp/backup/`, s3Path)
-	if s3Source.Endpoint != "" {
-		syncCmd = fmt.Sprintf(`aws s3 sync "s3://%s/" /tmp/backup/ --endpoint-url "%s"`, s3Path, s3Source.Endpoint)
-	}
-
-	restoreScript := fmt.Sprintf(`#!/bin/sh
-set -e
-echo "Downloading backup from S3..."
-%s
-RESTORE_DIR="/tmp/backup"
-if [ ! -f "${RESTORE_DIR}/manifest.json" ]; then
-  SUBDIR=$(find "${RESTORE_DIR}" -maxdepth 1 -mindepth 1 -type d | sort -r | head -1)
-  if [ -n "${SUBDIR}" ] && [ -f "${SUBDIR}/manifest.json" ]; then
-    RESTORE_DIR="${SUBDIR}"
-    echo "Found backup in subdirectory: ${RESTORE_DIR}"
-  fi
-fi
-echo "Restoring data to PVC..."
-hyperbytedb restore --input "${RESTORE_DIR}"
-echo "Restore complete for ordinal %d"
-`, syncCmd, ordinal)
-
-	env := []corev1.EnvVar{}
-	if s3Source.CredentialsSecretName != "" {
-		env = append(env,
-			corev1.EnvVar{
-				Name: "AWS_ACCESS_KEY_ID",
-				ValueFrom: &corev1.EnvVarSource{
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{Name: s3Source.CredentialsSecretName},
-						Key:                  "access_key_id",
-					},
-				},
-			},
-			corev1.EnvVar{
-				Name: "AWS_SECRET_ACCESS_KEY",
-				ValueFrom: &corev1.EnvVarSource{
-					SecretKeyRef: &corev1.SecretKeySelector{
-						LocalObjectReference: corev1.LocalObjectReference{Name: s3Source.CredentialsSecretName},
-						Key:                  "secret_access_key",
-					},
-				},
-			},
-		)
-	}
-	if s3Source.Region != "" {
-		env = append(env, corev1.EnvVar{Name: "AWS_DEFAULT_REGION", Value: s3Source.Region})
-	}
-
 	return &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-restore-%d", restore.Name, ordinal),
+			Name:      fmt.Sprintf("%s-%d", restore.Name, ordinal),
 			Namespace: restore.Namespace,
 			Labels:    hyperbytedb.CommonLabels(cluster),
 		},
@@ -312,27 +255,12 @@ echo "Restore complete for ordinal %d"
 			BackoffLimit: ptr.To(int32(2)),
 			Template: corev1.PodTemplateSpec{
 				Spec: corev1.PodSpec{
-					RestartPolicy: corev1.RestartPolicyOnFailure,
-					Containers: []corev1.Container{
-						{
-							Name:    "restore",
-							Image:   image,
-							Command: []string{"sh", "-c", restoreScript},
-							Env:     env,
-							VolumeMounts: []corev1.VolumeMount{
-								{Name: "data", MountPath: "/var/lib/hyperbytedb"},
-							},
-						},
-					},
+					RestartPolicy:  corev1.RestartPolicyOnFailure,
+					InitContainers: []corev1.Container{mcDownloadInitContainer(*s3Source)},
+					Containers:     []corev1.Container{restoreDataContainer(hyperbytedb.ResolveHyperbytedbImage(cluster), ordinal)},
 					Volumes: []corev1.Volume{
-						{
-							Name: "data",
-							VolumeSource: corev1.VolumeSource{
-								PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
-									ClaimName: fmt.Sprintf("data-%s-%d", cluster.Name, ordinal),
-								},
-							},
-						},
+						dataVolume(cluster.Name, ordinal, false),
+						snapshotVolume(),
 					},
 				},
 			},
diff --git a/internal/controller/s3_transfer.go b/internal/controller/s3_transfer.go
new file mode 100644
index 0000000..b9900fb
--- /dev/null
+++ b/internal/controller/s3_transfer.go
@@ -0,0 +1,171 @@
+package controller
+
+import (
+	"fmt"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+
+	hyperbytedbv1alpha1 "github.com/hyperbyte-cloud/hyperbytedb-operator/api/v1alpha1"
+)
+
+// Same tag Kind already pulls for the MinIO bucket job. The image includes /bin/sh.
+const defaultS3ClientImage = "minio/mc:RELEASE.2024-11-21T17-21-54Z"
+
+const (
+	snapshotVolumeName = "snapshot"
+	snapshotMountPath  = "/snapshot"
+	dataVolumeName     = "data"
+	dataMountPath      = "/var/lib/hyperbytedb"
+)
+
+func s3ClientImage() string {
+	return defaultS3ClientImage
+}
+
+func s3Endpoint(s3 hyperbytedbv1alpha1.S3BackupSpec) string {
+	if s3.Endpoint != "" {
+		return s3.Endpoint
+	}
+	if s3.Region != "" && s3.Region != "us-east-1" {
+		return "https://s3." + s3.Region + ".amazonaws.com"
+	}
+	return "https://s3.amazonaws.com"
+}
+
+func s3Prefix(s3 hyperbytedbv1alpha1.S3BackupSpec) string {
+	p := strings.Trim(s3.Prefix, "/")
+	if p == "" {
+		return ""
+	}
+	return p + "/"
+}
+
+func s3Env(s3 hyperbytedbv1alpha1.S3BackupSpec, extra ...corev1.EnvVar) []corev1.EnvVar {
+	env := []corev1.EnvVar{
+		{Name: "S3_ENDPOINT", Value: s3Endpoint(s3)},
+		{Name: "S3_BUCKET", Value: s3.Bucket},
+		{Name: "S3_PREFIX", Value: s3Prefix(s3)},
+	}
+	if s3.CredentialsSecretName != "" {
+		env = append(env,
+			corev1.EnvVar{
+				Name: "AWS_ACCESS_KEY_ID",
+				ValueFrom: &corev1.EnvVarSource{
+					SecretKeyRef: &corev1.SecretKeySelector{
+						LocalObjectReference: corev1.LocalObjectReference{Name: s3.CredentialsSecretName},
+						Key:                  "access_key_id",
+					},
+				},
+			},
+			corev1.EnvVar{
+				Name: "AWS_SECRET_ACCESS_KEY",
+				ValueFrom: &corev1.EnvVarSource{
+					SecretKeyRef: &corev1.SecretKeySelector{
+						LocalObjectReference: corev1.LocalObjectReference{Name: s3.CredentialsSecretName},
+						Key:                  "secret_access_key",
+					},
+				},
+			},
+		)
+	}
+	return append(env, extra...)
+}
+
+func snapshotVolume() corev1.Volume {
+	return corev1.Volume{
+		Name:         snapshotVolumeName,
+		VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}},
+	}
+}
+
+func dataVolume(clusterName string, ordinal int32, readOnly bool) corev1.Volume {
+	return corev1.Volume{
+		Name: dataVolumeName,
+		VolumeSource: corev1.VolumeSource{
+			PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+				ClaimName: fmt.Sprintf("data-%s-%d", clusterName, ordinal),
+				ReadOnly:  readOnly,
+			},
+		},
+	}
+}
+
+func snapshotBackupInitContainer(image string) corev1.Container {
+	return corev1.Container{
+		Name:  "snapshot",
+		Image: image,
+		Command: []string{"/bin/sh", "-ec", `echo "Starting hyperbytedb backup..."
+hyperbytedb backup --output /snapshot
+echo "Backup size: $(du -sh /snapshot | cut -f1)"
+`},
+		VolumeMounts: []corev1.VolumeMount{
+			{Name: dataVolumeName, MountPath: dataMountPath, ReadOnly: true},
+			{Name: snapshotVolumeName, MountPath: snapshotMountPath},
+		},
+	}
+}
+
+func mcUploadContainer(s3 hyperbytedbv1alpha1.S3BackupSpec, retentionDays int32) corev1.Container {
+	if retentionDays < 1 {
+		retentionDays = 7
+	}
+	script := fmt.Sprintf(`mc alias set dest "${S3_ENDPOINT}" "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}"
+TIMESTAMP=$(date +%%Y%%m%%d-%%H%%M%%S)
+DEST="dest/${S3_BUCKET}/${S3_PREFIX}${TIMESTAMP}"
+echo "Uploading to ${DEST}..."
+mc mirror /snapshot "${DEST}/"
+echo "Cleaning up backups older than %d days..."
+mc rm --recursive --force --older-than "%dd" "dest/${S3_BUCKET}/${S3_PREFIX}" || true
+echo "Backup complete"
+`, retentionDays, retentionDays)
+	return corev1.Container{
+		Name:    "upload",
+		Image:   s3ClientImage(),
+		Command: []string{"/bin/sh", "-ec", script},
+		Env:     s3Env(s3),
+		VolumeMounts: []corev1.VolumeMount{
+			{Name: snapshotVolumeName, MountPath: snapshotMountPath, ReadOnly: true},
+		},
+	}
+}
+
+func mcDownloadInitContainer(s3 hyperbytedbv1alpha1.S3BackupSpec) corev1.Container {
+	return corev1.Container{
+		Name:  "download",
+		Image: s3ClientImage(),
+		Command: []string{"/bin/sh", "-ec", `mc alias set dest "${S3_ENDPOINT}" "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}"
+SRC="dest/${S3_BUCKET}/${S3_PREFIX}"
+echo "Downloading backup from ${SRC}..."
+mc mirror "${SRC}" /snapshot/
+`},
+		Env: s3Env(s3),
+		VolumeMounts: []corev1.VolumeMount{
+			{Name: snapshotVolumeName, MountPath: snapshotMountPath},
+		},
+	}
+}
+
+func restoreDataContainer(image string, ordinal int32) corev1.Container {
+	script := fmt.Sprintf(`RESTORE_DIR="/snapshot"
+if [ ! -f "${RESTORE_DIR}/manifest.json" ]; then
+  SUBDIR=$(find "${RESTORE_DIR}" -maxdepth 1 -mindepth 1 -type d | sort -r | head -1)
+  if [ -n "${SUBDIR}" ] && [ -f "${SUBDIR}/manifest.json" ]; then
+    RESTORE_DIR="${SUBDIR}"
+    echo "Found backup in subdirectory: ${RESTORE_DIR}"
+  fi
+fi
+echo "Restoring data to PVC..."
+hyperbytedb restore --input "${RESTORE_DIR}"
+echo "Restore complete for ordinal %d"
+`, ordinal)
+	return corev1.Container{
+		Name:    "restore",
+		Image:   image,
+		Command: []string{"/bin/sh", "-ec", script},
+		VolumeMounts: []corev1.VolumeMount{
+			{Name: dataVolumeName, MountPath: dataMountPath},
+			{Name: snapshotVolumeName, MountPath: snapshotMountPath, ReadOnly: true},
+		},
+	}
+}