Skip to content

Upload backups with MinIO client instead of AWS CLI #14

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
austin-barrington merged 1 commit into from
Sep 4, 2026
Merged

Upload backups with MinIO client instead of AWS CLI #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 144 additions & 0 deletions internal/controller/backup_job_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
package controller

import (
"strings"
"testing"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"

hyperbytedbv1alpha1 "github.com/hyperbyte-cloud/hyperbytedb-operator/api/v1alpha1"
)

func testBackupCluster() *hyperbytedbv1alpha1.HyperbytedbCluster {
return &hyperbytedbv1alpha1.HyperbytedbCluster{
ObjectMeta: metav1.ObjectMeta{Name: "db-test", Namespace: "tenant-1"},
Spec: hyperbytedbv1alpha1.HyperbytedbClusterSpec{
Replicas: ptr.To(int32(1)),
Image: "ghcr.io/hyperbyte-cloud/hyperbytedb:0.8.5-beta",
},
}
}

func testBackup() *hyperbytedbv1alpha1.HyperbytedbBackup {
return &hyperbytedbv1alpha1.HyperbytedbBackup{
ObjectMeta: metav1.ObjectMeta{Name: "db-test-backup-abcd", Namespace: "tenant-1"},
Spec: hyperbytedbv1alpha1.HyperbytedbBackupSpec{
ClusterName: "db-test",
Destination: hyperbytedbv1alpha1.BackupDestination{
S3: hyperbytedbv1alpha1.S3BackupSpec{
Bucket: "hyperbytedb-backups",
Prefix: "inst-1",
Region: "us-east-1",
Endpoint: "http://minio.platform.svc.cluster.local:9000",
CredentialsSecretName: "hyperbytedb-backup-s3",
},
},
RetentionDays: 7,
},
}
}

func TestBuildBackupJobSnapshotsThenUploadsWithMc(t *testing.T) {
r := &HyperbytedbBackupReconciler{}
job := r.buildBackupJob(testBackup(), testBackupCluster(), "db-test-backup-abcd")
spec := job.Spec.Template.Spec

if len(spec.InitContainers) != 1 {
t.Fatalf("init containers = %d, want 1 (hyperbytedb snapshot)", len(spec.InitContainers))
}
init := spec.InitContainers[0]
if init.Name != "snapshot" {
t.Errorf("init name = %q, want snapshot", init.Name)
}
if init.Image != "ghcr.io/hyperbyte-cloud/hyperbytedb:0.8.5-beta" {
t.Errorf("snapshot image = %q", init.Image)
}
script := strings.Join(init.Command, " ")
if !strings.Contains(script, "hyperbytedb backup") {
t.Errorf("snapshot command missing hyperbytedb backup: %s", script)
}
if strings.Contains(script, "aws ") {
t.Errorf("snapshot must not call aws: %s", script)
}

if len(spec.Containers) != 1 {
t.Fatalf("containers = %d, want 1 (mc upload)", len(spec.Containers))
}
up := spec.Containers[0]
if up.Name != "upload" {
t.Errorf("container name = %q, want upload", up.Name)
}
if !strings.Contains(up.Image, "minio/mc") {
t.Errorf("upload image = %q, want minio/mc", up.Image)
}
upScript := strings.Join(up.Command, " ")
if !strings.Contains(upScript, "mc ") {
t.Errorf("upload command missing mc: %s", upScript)
}
if strings.Contains(upScript, "aws ") {
t.Errorf("upload must not call aws: %s", upScript)
}

names := map[string]bool{}
for _, v := range spec.Volumes {
names[v.Name] = true
}
if !names["snapshot"] || !names["data"] {
t.Errorf("volumes = %v, want snapshot emptyDir and data PVC", names)
}
}

func TestBuildRestoreJobDownloadsWithMcThenRestores(t *testing.T) {
r := &HyperbytedbRestoreReconciler{}
restore := &hyperbytedbv1alpha1.HyperbytedbRestore{
ObjectMeta: metav1.ObjectMeta{Name: "db-test-restore-abcd", Namespace: "tenant-1"},
Spec: hyperbytedbv1alpha1.HyperbytedbRestoreSpec{
ClusterName: "db-test",
BackupName: "db-test-backup-abcd",
},
}
s3 := &hyperbytedbv1alpha1.S3BackupSpec{
Bucket: "hyperbytedb-backups",
Prefix: "inst-1",
Endpoint: "http://minio.platform.svc.cluster.local:9000",
CredentialsSecretName: "hyperbytedb-backup-s3",
}
job := r.buildRestoreJob(restore, testBackupCluster(), s3, 0)
spec := job.Spec.Template.Spec

if len(spec.InitContainers) != 1 {
t.Fatalf("init containers = %d, want 1 (mc download)", len(spec.InitContainers))
}
dl := spec.InitContainers[0]
if dl.Name != "download" {
t.Errorf("init name = %q, want download", dl.Name)
}
if !strings.Contains(dl.Image, "minio/mc") {
t.Errorf("download image = %q, want minio/mc", dl.Image)
}
if strings.Contains(strings.Join(dl.Command, " "), "aws ") {
t.Errorf("download must not call aws: %s", dl.Command)
}

if len(spec.Containers) != 1 {
t.Fatalf("containers = %d, want 1 (hyperbytedb restore)", len(spec.Containers))
}
rst := spec.Containers[0]
if rst.Image != "ghcr.io/hyperbyte-cloud/hyperbytedb:0.8.5-beta" {
t.Errorf("restore image = %q", rst.Image)
}
script := strings.Join(rst.Command, " ")
if !strings.Contains(script, "hyperbytedb restore") {
t.Errorf("restore command missing hyperbytedb restore: %s", script)
}
if strings.Contains(script, "aws ") {
t.Errorf("restore must not call aws: %s", script)
}

restore.Name = "db-7f4eaf37-8805-46e4-97d2-6a4c7d18c0e2-restore-3f839e85"
job = r.buildRestoreJob(restore, testBackupCluster(), s3, 0)
if n := len(job.Name); n > 63 {
t.Fatalf("restore job name %q is %d bytes; Kubernetes job-name labels must be <=63", job.Name, n)
}
}
113 changes: 5 additions & 108 deletions internal/controller/hyperbytedbbackup_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,94 +207,7 @@ func (r *HyperbytedbBackupReconciler) reconcileCronJob(ctx context.Context, back
}

func (r *HyperbytedbBackupReconciler) buildBackupJob(backup *hyperbytedbv1alpha1.HyperbytedbBackup, cluster *hyperbytedbv1alpha1.HyperbytedbCluster, name string) *batchv1.Job {
image := hyperbytedb.ResolveHyperbytedbImage(cluster)

s3 := backup.Spec.Destination.S3
s3Path := r.buildBackupS3Path(backup)

backupScript := fmt.Sprintf(`#!/bin/sh
set -e
BACKUP_DIR="/tmp/backup"
TIMESTAMP=$(date +%%Y%%m%%d-%%H%%M%%S)
S3_DEST="s3://%s/${TIMESTAMP}/"

echo "Starting hyperbytedb backup..."
hyperbytedb backup --output "${BACKUP_DIR}"

BACKUP_SIZE=$(du -sh "${BACKUP_DIR}" | cut -f1)
echo "Backup size: ${BACKUP_SIZE}"

echo "Uploading to ${S3_DEST}..."
`, s3Path)

if s3.Endpoint != "" {
backupScript += fmt.Sprintf(`aws s3 sync "${BACKUP_DIR}" "${S3_DEST}" --endpoint-url "%s"
`, s3.Endpoint)
} else {
backupScript += `aws s3 sync "${BACKUP_DIR}" "${S3_DEST}"
`
}

backupScript += fmt.Sprintf(`
echo "Cleaning up backups older than %d days..."
`, backup.Spec.RetentionDays)

if s3.Endpoint != "" {
backupScript += fmt.Sprintf(`CUTOFF=$(date -d "-%d days" +%%Y%%m%%d-000000 2>/dev/null || date -v -%dd +%%Y%%m%%d-000000 2>/dev/null || echo "")
if [ -n "$CUTOFF" ]; then
aws s3 ls "s3://%s/" --endpoint-url "%s" | while read -r line; do
DIR=$(echo "$line" | awk '{print $NF}' | tr -d '/')
if [ "$DIR" \< "$CUTOFF" ] 2>/dev/null; then
echo "Removing old backup: ${DIR}"
aws s3 rm "s3://%s/${DIR}/" --recursive --endpoint-url "%s"
fi
done
fi
`, backup.Spec.RetentionDays, backup.Spec.RetentionDays, s3Path, s3.Endpoint, s3Path, s3.Endpoint)
} else {
backupScript += fmt.Sprintf(`CUTOFF=$(date -d "-%d days" +%%Y%%m%%d-000000 2>/dev/null || date -v -%dd +%%Y%%m%%d-000000 2>/dev/null || echo "")
if [ -n "$CUTOFF" ]; then
aws s3 ls "s3://%s/" | while read -r line; do
DIR=$(echo "$line" | awk '{print $NF}' | tr -d '/')
if [ "$DIR" \< "$CUTOFF" ] 2>/dev/null; then
echo "Removing old backup: ${DIR}"
aws s3 rm "s3://%s/${DIR}/" --recursive
fi
done
fi
`, backup.Spec.RetentionDays, backup.Spec.RetentionDays, s3Path, s3Path)
}

backupScript += `echo "Backup complete"
`

env := []corev1.EnvVar{}
if s3.CredentialsSecretName != "" {
env = append(env,
corev1.EnvVar{
Name: "AWS_ACCESS_KEY_ID",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: s3.CredentialsSecretName},
Key: "access_key_id",
},
},
},
corev1.EnvVar{
Name: "AWS_SECRET_ACCESS_KEY",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: s3.CredentialsSecretName},
Key: "secret_access_key",
},
},
},
)
}
if s3.Region != "" {
env = append(env, corev1.EnvVar{Name: "AWS_DEFAULT_REGION", Value: s3.Region})
}

return &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Expand All @@ -305,28 +218,12 @@ fi
BackoffLimit: ptr.To(int32(3)),
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyOnFailure,
Containers: []corev1.Container{
{
Name: "backup",
Image: image,
Command: []string{"sh", "-c", backupScript},
Env: env,
VolumeMounts: []corev1.VolumeMount{
{Name: "data", MountPath: "/var/lib/hyperbytedb", ReadOnly: true},
},
},
},
RestartPolicy: corev1.RestartPolicyOnFailure,
InitContainers: []corev1.Container{snapshotBackupInitContainer(hyperbytedb.ResolveHyperbytedbImage(cluster))},
Containers: []corev1.Container{mcUploadContainer(s3, backup.Spec.RetentionDays)},
Volumes: []corev1.Volume{
{
Name: "data",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: fmt.Sprintf("data-%s-0", cluster.Name),
ReadOnly: true,
},
},
},
dataVolume(cluster.Name, 0, true),
snapshotVolume(),
},
},
},
Expand Down
86 changes: 7 additions & 79 deletions internal/controller/hyperbytedbrestore_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package controller
import (
"context"
"fmt"
"strings"
"time"

appsv1 "k8s.io/api/apps/v1"
Expand Down Expand Up @@ -187,7 +186,7 @@ func (r *HyperbytedbRestoreReconciler) checkRestoreJob(ctx context.Context, rest
anyFailed := false

for i := int32(0); i < replicas; i++ {
jobName := fmt.Sprintf("%s-restore-%d", restore.Name, i)
jobName := fmt.Sprintf("%s-%d", restore.Name, i)
job := &batchv1.Job{}
if err := r.Get(ctx, types.NamespacedName{Name: jobName, Namespace: restore.Namespace}, job); err != nil {
if apierrors.IsNotFound(err) {
Expand Down Expand Up @@ -246,93 +245,22 @@ func (r *HyperbytedbRestoreReconciler) buildRestoreJob(
s3Source *hyperbytedbv1alpha1.S3BackupSpec,
ordinal int32,
) *batchv1.Job {
image := hyperbytedb.ResolveHyperbytedbImage(cluster)

s3Path := s3Source.Bucket
if s3Source.Prefix != "" {
s3Path += "/" + strings.TrimSuffix(s3Source.Prefix, "/")
}

syncCmd := fmt.Sprintf(`aws s3 sync "s3://%s/" /tmp/backup/`, s3Path)
if s3Source.Endpoint != "" {
syncCmd = fmt.Sprintf(`aws s3 sync "s3://%s/" /tmp/backup/ --endpoint-url "%s"`, s3Path, s3Source.Endpoint)
}

restoreScript := fmt.Sprintf(`#!/bin/sh
set -e
echo "Downloading backup from S3..."
%s
RESTORE_DIR="/tmp/backup"
if [ ! -f "${RESTORE_DIR}/manifest.json" ]; then
SUBDIR=$(find "${RESTORE_DIR}" -maxdepth 1 -mindepth 1 -type d | sort -r | head -1)
if [ -n "${SUBDIR}" ] && [ -f "${SUBDIR}/manifest.json" ]; then
RESTORE_DIR="${SUBDIR}"
echo "Found backup in subdirectory: ${RESTORE_DIR}"
fi
fi
echo "Restoring data to PVC..."
hyperbytedb restore --input "${RESTORE_DIR}"
echo "Restore complete for ordinal %d"
`, syncCmd, ordinal)

env := []corev1.EnvVar{}
if s3Source.CredentialsSecretName != "" {
env = append(env,
corev1.EnvVar{
Name: "AWS_ACCESS_KEY_ID",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: s3Source.CredentialsSecretName},
Key: "access_key_id",
},
},
},
corev1.EnvVar{
Name: "AWS_SECRET_ACCESS_KEY",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: s3Source.CredentialsSecretName},
Key: "secret_access_key",
},
},
},
)
}
if s3Source.Region != "" {
env = append(env, corev1.EnvVar{Name: "AWS_DEFAULT_REGION", Value: s3Source.Region})
}

return &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-restore-%d", restore.Name, ordinal),
Name: fmt.Sprintf("%s-%d", restore.Name, ordinal),
Namespace: restore.Namespace,
Labels: hyperbytedb.CommonLabels(cluster),
},
Spec: batchv1.JobSpec{
BackoffLimit: ptr.To(int32(2)),
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyOnFailure,
Containers: []corev1.Container{
{
Name: "restore",
Image: image,
Command: []string{"sh", "-c", restoreScript},
Env: env,
VolumeMounts: []corev1.VolumeMount{
{Name: "data", MountPath: "/var/lib/hyperbytedb"},
},
},
},
RestartPolicy: corev1.RestartPolicyOnFailure,
InitContainers: []corev1.Container{mcDownloadInitContainer(*s3Source)},
Containers: []corev1.Container{restoreDataContainer(hyperbytedb.ResolveHyperbytedbImage(cluster), ordinal)},
Volumes: []corev1.Volume{
{
Name: "data",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: fmt.Sprintf("data-%s-%d", cluster.Name, ordinal),
},
},
},
dataVolume(cluster.Name, ordinal, false),
snapshotVolume(),
},
},
},
Expand Down
Loading
Loading