From 329ee14aaa53d73440439bff94abe384a103bd38 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Wed, 23 Sep 2026 12:31:59 +0200 Subject: [PATCH] Settle incoming migrations before declaring a hypervisor evicted During a live migration, Nova can still associate the instance with its source while it is already moving to the destination. Relying only on the destination's instance count can therefore declare a host evicted too early. Require incoming migrations to settle before completing eviction, and make an unsuccessful Nova check visible without allowing eviction to continue. Once persisted state conclusively shows that the host is empty and settled, avoid repeatedly verifying the same terminal state. Signed-off-by: Fabian Wiesel --- api/v1/eviction_types.go | 16 + .../hypervisor_maintenance_controller.go | 190 +++++- .../hypervisor_maintenance_controller_test.go | 412 ++++++++++++ internal/openstack/migrations.go | 173 +++++ internal/openstack/migrations_test.go | 618 ++++++++++++++++++ 5 files changed, 1383 insertions(+), 26 deletions(-) create mode 100644 internal/openstack/migrations.go create mode 100644 internal/openstack/migrations_test.go diff --git a/api/v1/eviction_types.go b/api/v1/eviction_types.go index b96ad007..5f6039f2 100644 --- a/api/v1/eviction_types.go +++ b/api/v1/eviction_types.go @@ -53,6 +53,13 @@ const ( // ConditionTypeHypervisorDisabled is the type of condition for hypervisor disabled status ConditionTypeHypervisorDisabled = "HypervisorDisabled" + // ConditionTypeIncomingMigrationsSettled is set on the Hypervisor CR by the + // maintenance controller. True means no live-migrations or evacuations are + // currently in flight targeting this host. False/Unknown/Missing means the + // controller has observed outstanding incoming migrations (or hasn't checked yet) + // and the host must not be declared evicted. + ConditionTypeIncomingMigrationsSettled = "IncomingMigrationsSettled" + // ConditionTypeHaEnabled is the type of condition for signalling if HA is enabled / disabled for the hypervisor ConditionTypeHaEnabled = "HaEnabled" @@ -70,6 +77,15 @@ const ( // ConditionReasonSucceeded means the eviction has succeeded ConditionReasonSucceeded string = "Succeeded" + + // ConditionReasonSettled means all incoming migrations have reached a terminal state + ConditionReasonSettled string = "Settled" + + // ConditionReasonAborting means the controller is aborting incoming migrations + ConditionReasonAborting string = "AbortingIncomingMigrations" + + // ConditionReasonWaiting means the controller is waiting for non-abortable incoming migrations + ConditionReasonWaiting string = "WaitingForIncomingMigrations" ) // EvictionStatus defines the observed state of Eviction diff --git a/internal/controller/hypervisor_maintenance_controller.go b/internal/controller/hypervisor_maintenance_controller.go index 7edeb503..b03b8f89 100644 --- a/internal/controller/hypervisor_maintenance_controller.go +++ b/internal/controller/hypervisor_maintenance_controller.go @@ -24,6 +24,7 @@ import ( "context" "errors" "fmt" + "time" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -44,6 +45,10 @@ import ( const ( HypervisorMaintenanceControllerName = "HypervisorMaintenance" + + // settleRequeueInterval is the interval at which the controller re-checks + // incoming migrations when they haven't settled yet. + settleRequeueInterval = 10 * time.Second ) type HypervisorMaintenanceController struct { @@ -72,27 +77,33 @@ func (hec *HypervisorMaintenanceController) Reconcile(ctx context.Context, req c // Include unchanged fields owned by this controller because omitting them // from a subsequent SSA payload would prune them. Seed only the // HypervisorDisabled condition here because it is always retained. - // reconcileEviction conditionally seeds ConditionTypeEvicting: it is included - // when maintenance is active, and intentionally omitted when MaintenanceUnset - // so that SSA prunes it from the field manager's managed fields. + // reconcileEviction conditionally seeds ConditionTypeEvicting and + // ConditionTypeIncomingMigrationsSettled: they are included when maintenance + // is active, and intentionally omitted when MaintenanceUnset so that SSA + // prunes them from the field manager's managed fields. statusCfg := apiv1.HypervisorStatus().WithEvicted(hv.Status.Evicted) retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled) if err := hec.reconcileComputeService(ctx, hv, statusCfg); err != nil { retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeEvicting) + retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) applyErr := hec.Status().Apply(ctx, apiv1.Hypervisor(hv.Name).WithStatus(statusCfg), k8sclient.ForceOwnership, k8sclient.FieldOwner(HypervisorMaintenanceControllerName)) return ctrl.Result{}, errors.Join(err, applyErr) } - if err := hec.reconcileEviction(ctx, hv, statusCfg); err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{}, hec.Status().Apply(ctx, + requeue, err := hec.reconcileEviction(ctx, hv, statusCfg) + applyErr := hec.Status().Apply(ctx, apiv1.Hypervisor(hv.Name).WithStatus(statusCfg), k8sclient.ForceOwnership, k8sclient.FieldOwner(HypervisorMaintenanceControllerName)) + if err != nil || applyErr != nil { + return ctrl.Result{}, errors.Join(err, applyErr) + } + if requeue { + return ctrl.Result{RequeueAfter: settleRequeueInterval}, nil + } + return ctrl.Result{}, nil } // retainStatusCondition keeps an existing condition of the given type in the @@ -181,9 +192,10 @@ func (hec *HypervisorMaintenanceController) reconcileComputeService(ctx context. } // reconcileEviction creates/deletes the Eviction CR and sets the ConditionTypeEvicting -// condition and Evicted scalar on statusCfg. When eviction should be removed, the -// condition entry is filtered out so SSA prunes it. -func (hec *HypervisorMaintenanceController) reconcileEviction(ctx context.Context, hv *kvmv1.Hypervisor, statusCfg *apiv1.HypervisorStatusApplyConfiguration) error { +// and ConditionTypeIncomingMigrationsSettled conditions plus the Evicted scalar on +// statusCfg. When eviction should be removed, the condition entries are filtered out +// so SSA prunes them. +func (hec *HypervisorMaintenanceController) reconcileEviction(ctx context.Context, hv *kvmv1.Hypervisor, statusCfg *apiv1.HypervisorStatusApplyConfiguration) (requeue bool, err error) { eviction := &kvmv1.Eviction{ Name: hv.Name, } @@ -191,35 +203,102 @@ func (hec *HypervisorMaintenanceController) reconcileEviction(ctx context.Contex switch hv.Spec.Maintenance { case kvmv1.MaintenanceUnset: // Avoid deleting the eviction over and over. - if !hv.Status.Evicted && meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeEvicting) == nil { - return nil + if !hv.Status.Evicted && + meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeEvicting) == nil && + meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) == nil { + return false, nil } if err := k8sclient.IgnoreNotFound(hec.Delete(ctx, eviction)); err != nil { - return err + return false, err } - // ConditionTypeEvicting is intentionally absent from statusCfg — SSA - // will prune it from this field manager's managed fields on Apply. + // ConditionTypeEvicting and ConditionTypeIncomingMigrationsSettled are + // intentionally absent from statusCfg — SSA will prune them from this + // field manager's managed fields on Apply. statusCfg.WithEvicted(false) case kvmv1.MaintenanceManual, kvmv1.MaintenanceAuto, kvmv1.MaintenanceTermination: - // In case of "ha", the host gets emptied from the HA service. - // Seed the existing evicting condition so SSA does not prune it, - // regardless of whether we take the short-circuit below. - if cond := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeEvicting); cond != nil { - statusCfg.WithConditions(utils.ConditionFromStatus(*cond)) - if cond.Reason == kvmv1.ConditionReasonSucceeded { - // We are done here, no need to look at the eviction any more. - return nil + evictingCondition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeEvicting) + migrationsCondition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) + if evictingCondition != nil && + evictingCondition.Status == metav1.ConditionFalse && + evictingCondition.Reason == kvmv1.ConditionReasonSucceeded { + if hv.Status.NumInstances > 0 { + // Instance appeared after eviction completed. Delete the Eviction CR so + // ensureEviction recreates a fresh one. + log := logger.FromContext(ctx) + log.Info("Eviction reported succeeded but instances remain on host; re-entering drain", + "numInstances", hv.Status.NumInstances) + if delErr := k8sclient.IgnoreNotFound(hec.Delete(ctx, eviction)); delErr != nil { + return false, delErr + } + statusCfg.WithEvicted(false) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeEvicting). + WithStatus(metav1.ConditionTrue). + WithReason(kvmv1.ConditionReasonRunning). + WithMessage(fmt.Sprintf("Re-entering drain: %d instance(s) still on host", hv.Status.NumInstances))) + retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) + return false, nil + } + + if hv.Status.Evicted && migrationsCondition != nil && + migrationsCondition.Status == metav1.ConditionTrue && + migrationsCondition.Reason == kvmv1.ConditionReasonSettled { + retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeEvicting) + retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) + return false, nil } } + // Gate: ensure all incoming migrations are settled before proceeding. + settled, err := hec.settleIncomingMigrations(ctx, hv, statusCfg) + if err != nil { + statusCfg.WithEvicted(false) + retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeEvicting) + return false, err + } + if !settled { + // Cannot proceed with eviction while incoming migrations are outstanding. + // Clear the terminal scalar and seed the existing evicting condition so + // SSA does not leave the hypervisor marked as evicted or prune it. + statusCfg.WithEvicted(false) + retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeEvicting) + return true, nil + } + + if evictingCondition != nil && evictingCondition.Reason == kvmv1.ConditionReasonSucceeded { + // The host was not previously known to be settled, but the current Nova + // check established it. Preserve the completed eviction state. + retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeEvicting) + return false, nil + } + status, err := hec.ensureEviction(ctx, eviction, hv) if err != nil { - return err + return false, err } var reason, message string if status == metav1.ConditionFalse { + if hv.Status.NumInstances > 0 { + // Eviction CR says done but instances exist (race with in-flight migration). + // Delete the Eviction CR and restart drain. + log := logger.FromContext(ctx) + log.Info("Eviction finished but instances remain on host; restarting drain", + "numInstances", hv.Status.NumInstances) + if delErr := k8sclient.IgnoreNotFound(hec.Delete(ctx, eviction)); delErr != nil { + return false, delErr + } + statusCfg.WithEvicted(false) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeEvicting). + WithStatus(metav1.ConditionTrue). + WithReason(kvmv1.ConditionReasonRunning). + WithMessage(fmt.Sprintf("Restarting drain: %d instance(s) still on host", hv.Status.NumInstances))) + return false, nil + } message = "Evicted" reason = kvmv1.ConditionReasonSucceeded statusCfg.WithEvicted(true) @@ -237,7 +316,66 @@ func (hec *HypervisorMaintenanceController) reconcileEviction(ctx context.Contex WithMessage(message)) } - return nil + return false, nil +} + +// settleIncomingMigrations checks for in-flight migrations targeting this host, +// aborts those that can be aborted, and reports whether the host is settled. +// It sets the IncomingMigrationsSettled condition on statusCfg. +// Returns true if settled (no incoming migrations), false otherwise. +func (hec *HypervisorMaintenanceController) settleIncomingMigrations(ctx context.Context, hv *kvmv1.Hypervisor, statusCfg *apiv1.HypervisorStatusApplyConfiguration) (bool, error) { + log := logger.FromContext(ctx) + + aborted, waiting, err := openstack.SettleIncomingMigrations(ctx, hec.computeClient, hv.Name) + if err != nil { + err = fmt.Errorf("settling incoming migrations for %s: %w", hv.Name, err) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeIncomingMigrationsSettled). + WithStatus(metav1.ConditionUnknown). + WithReason(kvmv1.ConditionReasonFailed). + WithMessage(err.Error())) + return false, err + } + + if len(waiting) > 0 { + instanceUUIDs := make([]string, len(waiting)) + for i, m := range waiting { + instanceUUIDs[i] = m.InstanceUUID + } + log.Info("Waiting for non-abortable incoming migrations", "host", hv.Name, "instances", instanceUUIDs) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeIncomingMigrationsSettled). + WithStatus(metav1.ConditionFalse). + WithReason(kvmv1.ConditionReasonWaiting). + WithMessage(fmt.Sprintf("Waiting for %d non-abortable incoming migration(s) to complete", len(waiting)))) + return false, nil + } + + if len(aborted) > 0 { + instanceUUIDs := make([]string, len(aborted)) + for i, m := range aborted { + instanceUUIDs[i] = m.InstanceUUID + } + log.Info("Aborted incoming migrations", "host", hv.Name, "instances", instanceUUIDs) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeIncomingMigrationsSettled). + WithStatus(metav1.ConditionFalse). + WithReason(kvmv1.ConditionReasonAborting). + WithMessage(fmt.Sprintf("Aborted %d incoming migration(s); verifying on next reconcile", len(aborted)))) + return false, nil + } + + // No incoming migrations — settled. + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeIncomingMigrationsSettled). + WithStatus(metav1.ConditionTrue). + WithReason(kvmv1.ConditionReasonSettled). + WithMessage("No incoming migrations targeting this host")) + return true, nil } func (hec *HypervisorMaintenanceController) ensureEviction(ctx context.Context, eviction *kvmv1.Eviction, hypervisor *kvmv1.Hypervisor) (metav1.ConditionStatus, error) { diff --git a/internal/controller/hypervisor_maintenance_controller_test.go b/internal/controller/hypervisor_maintenance_controller_test.go index 27990d44..af7d5e75 100644 --- a/internal/controller/hypervisor_maintenance_controller_test.go +++ b/internal/controller/hypervisor_maintenance_controller_test.go @@ -20,6 +20,7 @@ package controller import ( "fmt" "net/http" + "time" "github.com/gophercloud/gophercloud/v2/testhelper" "github.com/gophercloud/gophercloud/v2/testhelper/client" @@ -28,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + applymetav1 "k8s.io/client-go/applyconfigurations/meta/v1" ctrl "sigs.k8s.io/controller-runtime" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -43,6 +45,10 @@ var _ = Describe("HypervisorMaintenanceController", func() { hypervisorName = types.NamespacedName{Name: "hv-test"} expectReconcileErr bool reconcileErr error + migrationsResponse string // mutable: tests can set this to control /os-migrations responses + deletedMigrations []string + migrationRequests int + migrationsStatus int ) const ( @@ -88,6 +94,35 @@ var _ = Describe("HypervisorMaintenanceController", func() { expectReconcileErr = false reconcileErr = nil + // Default: no incoming migrations + migrationsResponse = `{"migrations": []}` + deletedMigrations = nil + migrationRequests = 0 + migrationsStatus = http.StatusOK + + fakeServer.Mux.HandleFunc("GET /os-migrations", func(w http.ResponseWriter, r *http.Request) { + migrationRequests++ + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(migrationsStatus) + if migrationsStatus != http.StatusOK { + fmt.Fprint(w, `{"error": "nova unavailable"}`) + return + } + mt := r.URL.Query().Get("migration_type") + // Return the configured migrations for the default query; + // evacuation queries return empty by default. + if mt == "evacuation" { + fmt.Fprint(w, `{"migrations": []}`) + } else { + fmt.Fprint(w, migrationsResponse) + } + }) + + fakeServer.Mux.HandleFunc("DELETE /servers/", func(w http.ResponseWriter, r *http.Request) { + deletedMigrations = append(deletedMigrations, r.URL.Path) + w.WriteHeader(http.StatusAccepted) + }) + By("Creating the HypervisorMaintenanceController") controller = &HypervisorMaintenanceController{ Client: k8sClient, @@ -186,6 +221,12 @@ var _ = Describe("HypervisorMaintenanceController", func() { Reason: kvmv1.ConditionReasonRunning, Message: "Evicting", }) + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeIncomingMigrationsSettled, + Status: metav1.ConditionFalse, + Reason: kvmv1.ConditionReasonWaiting, + Message: "Waiting for incoming migrations", + }) Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) hypervisor.Spec.Maintenance = kvmv1.MaintenanceAuto @@ -208,6 +249,9 @@ var _ = Describe("HypervisorMaintenanceController", func() { evictingCondition := meta.FindStatusCondition(updated.Status.Conditions, kvmv1.ConditionTypeEvicting) Expect(evictingCondition).NotTo(BeNil()) Expect(evictingCondition.Status).To(Equal(metav1.ConditionTrue)) + migrationsCondition := meta.FindStatusCondition(updated.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) + Expect(migrationsCondition).NotTo(BeNil()) + Expect(migrationsCondition.Status).To(Equal(metav1.ConditionFalse)) eviction := &kvmv1.Eviction{} err := k8sClient.Get(ctx, hypervisorName, eviction) @@ -312,6 +356,42 @@ var _ = Describe("HypervisorMaintenanceController", func() { }) }) // Spec.Maintenance="" + Context("Spec.Maintenance=\"\" with only stale IncomingMigrationsSettled condition", func() { + BeforeEach(func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + + // First, simulate that the controller previously owned the condition + // by applying it via SSA with the same field manager. + statusCfg := applyv1.HypervisorStatus(). + WithEvicted(false). + WithConditions(applymetav1.Condition(). + WithType(kvmv1.ConditionTypeIncomingMigrationsSettled). + WithStatus(metav1.ConditionFalse). + WithReason(kvmv1.ConditionReasonWaiting). + WithMessage("incoming migrations still pending"). + WithLastTransitionTime(metav1.Now())) + Expect(k8sClient.Status().Apply(ctx, + applyv1.Hypervisor(hypervisor.Name).WithStatus(statusCfg), + k8sclient.ForceOwnership, + k8sclient.FieldOwner(HypervisorMaintenanceControllerName), + )).To(Succeed()) + + // Now cancel maintenance. + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + hypervisor.Spec.Maintenance = "" + Expect(k8sClient.Update(ctx, hypervisor)).To(Succeed()) + expectedBody := `{"status": "enabled", "forced_down": false}` + mockServiceUpdate(expectedBody) + }) + + It("should remove the stale IncomingMigrationsSettled condition", func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + Expect(meta.FindStatusCondition(hypervisor.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled)).To(BeNil()) + }) + }) + Context("Spec.Maintenance=\"ha\"", func() { BeforeEach(func(ctx SpecContext) { hypervisor := &kvmv1.Hypervisor{} @@ -556,6 +636,338 @@ var _ = Describe("HypervisorMaintenanceController", func() { Expect(hypervisor.Status.ServiceID).To(BeEmpty()) }) }) + + Context("Incoming migrations settling", func() { + BeforeEach(func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + hypervisor.Status.ServiceID = "1234" + meta.SetStatusCondition(&hypervisor.Status.Conditions, + metav1.Condition{ + Type: kvmv1.ConditionTypeOnboarding, + Status: metav1.ConditionFalse, + Reason: metav1.StatusSuccess, + Message: "Onboarded", + }, + ) + Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) + + // Re-read to get fresh resourceVersion, then update spec + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + hypervisor.Spec.Maintenance = "auto" + Expect(k8sClient.Update(ctx, hypervisor)).To(Succeed()) + + // Permissive service mock: accept any enable/disable call + fakeServer.Mux.HandleFunc("PUT /os-services/1234", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, ServiceEnabledResponse) + }) + }) + + Context("when there are no incoming migrations", func() { + It("should set IncomingMigrationsSettled=True and proceed to create eviction", func(ctx SpecContext) { + Expect(migrationRequests).To(Equal(1)) + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + Expect(meta.IsStatusConditionTrue(hypervisor.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled)).To(BeTrue()) + // Eviction should be created + eviction := &kvmv1.Eviction{} + Expect(k8sClient.Get(ctx, hypervisorName, eviction)).To(Succeed()) + }) + }) + + Context("when Nova cannot list incoming migrations", func() { + BeforeEach(func(ctx SpecContext) { + expectReconcileErr = true + migrationsStatus = http.StatusServiceUnavailable + + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + hypervisor.Status.Evicted = true + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeEvicting, + Status: metav1.ConditionTrue, + Reason: kvmv1.ConditionReasonRunning, + Message: "Evicting", + }) + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeIncomingMigrationsSettled, + Status: metav1.ConditionTrue, + Reason: kvmv1.ConditionReasonSettled, + Message: "No incoming migrations targeting this host", + }) + Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) + }) + + It("reports that migration settlement is unknown", func(ctx SpecContext) { + Expect(reconcileErr).To(MatchError(ContainSubstring("settling incoming migrations"))) + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + condition := meta.FindStatusCondition(hypervisor.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) + Expect(condition).NotTo(BeNil()) + Expect(condition.Status).To(Equal(metav1.ConditionUnknown)) + Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonFailed)) + Expect(condition.Message).To(ContainSubstring("nova unavailable")) + Expect(hypervisor.Status.Evicted).To(BeFalse()) + evictingCondition := meta.FindStatusCondition(hypervisor.Status.Conditions, kvmv1.ConditionTypeEvicting) + Expect(evictingCondition).NotTo(BeNil()) + Expect(evictingCondition.Status).To(Equal(metav1.ConditionTrue)) + Expect(evictingCondition.Reason).To(Equal(kvmv1.ConditionReasonRunning)) + }) + + It("does not create an eviction", func(ctx SpecContext) { + eviction := &kvmv1.Eviction{} + err := k8sClient.Get(ctx, hypervisorName, eviction) + Expect(err).To(HaveOccurred()) + Expect(k8sclient.IgnoreNotFound(err)).To(Succeed()) + }) + }) + + Context("when the host is conclusively evicted", func() { + BeforeEach(func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + hypervisor.Status.Evicted = true + hypervisor.Status.NumInstances = 0 + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeEvicting, + Status: metav1.ConditionFalse, + Reason: kvmv1.ConditionReasonSucceeded, + Message: "Evicted", + }) + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeIncomingMigrationsSettled, + Status: metav1.ConditionTrue, + Reason: kvmv1.ConditionReasonSettled, + Message: "No incoming migrations targeting this host", + }) + Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) + }) + + It("does not query Nova again", func(ctx SpecContext) { + Expect(migrationRequests).To(BeZero()) + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + Expect(hypervisor.Status.Evicted).To(BeTrue()) + Expect(meta.IsStatusConditionTrue(hypervisor.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled)).To(BeTrue()) + }) + }) + + Context("when there is a running incoming migration", func() { + BeforeEach(func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + hypervisor.Status.Evicted = true + Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) + + migrationsResponse = `{"migrations": [ + { + "id": 42, + "uuid": "mig-uuid-1", + "instance_uuid": "inst-uuid-1", + "status": "running", + "source_compute": "node003", + "dest_compute": "hv-test", + "migration_type": "live-migration" + } + ]}` + }) + + It("should set IncomingMigrationsSettled=False with Aborting reason", func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + cond := meta.FindStatusCondition(hypervisor.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + Expect(cond.Reason).To(Equal(kvmv1.ConditionReasonAborting)) + }) + + It("should clear a previously reported evicted status", func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + Expect(hypervisor.Status.Evicted).To(BeFalse()) + }) + + It("should issue a DELETE to abort the migration", func(_ SpecContext) { + Expect(deletedMigrations).To(HaveLen(1)) + Expect(deletedMigrations[0]).To(ContainSubstring("inst-uuid-1")) + Expect(deletedMigrations[0]).To(ContainSubstring("42")) + }) + + It("should not create an eviction resource", func(ctx SpecContext) { + eviction := &kvmv1.Eviction{} + err := k8sClient.Get(ctx, hypervisorName, eviction) + Expect(err).To(HaveOccurred()) + Expect(k8sclient.IgnoreNotFound(err)).To(Succeed()) + }) + + It("should requeue after settleRequeueInterval", func(ctx SpecContext) { + req := ctrl.Request{NamespacedName: hypervisorName} + result, err := controller.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(Equal(10 * time.Second)) + }) + }) + + Context("when there is a post-migrating incoming migration", func() { + BeforeEach(func(_ SpecContext) { + migrationsResponse = `{"migrations": [ + { + "id": 99, + "uuid": "mig-uuid-2", + "instance_uuid": "inst-uuid-2", + "status": "post-migrating", + "source_compute": "node003", + "dest_compute": "hv-test", + "migration_type": "live-migration" + } + ]}` + }) + + It("should set IncomingMigrationsSettled=False with Waiting reason", func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + cond := meta.FindStatusCondition(hypervisor.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + Expect(cond.Reason).To(Equal(kvmv1.ConditionReasonWaiting)) + }) + + It("should not issue any DELETE", func(_ SpecContext) { + Expect(deletedMigrations).To(BeEmpty()) + }) + + It("should not create an eviction resource", func(ctx SpecContext) { + eviction := &kvmv1.Eviction{} + err := k8sClient.Get(ctx, hypervisorName, eviction) + Expect(err).To(HaveOccurred()) + Expect(k8sclient.IgnoreNotFound(err)).To(Succeed()) + }) + + It("should requeue after settleRequeueInterval", func(ctx SpecContext) { + req := ctrl.Request{NamespacedName: hypervisorName} + result, err := controller.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(Equal(10 * time.Second)) + }) + }) + + Context("when eviction succeeded but instances remain (incident regression)", func() { + BeforeEach(func(ctx SpecContext) { + // Simulate: eviction CR exists and reports Succeeded + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + + eviction := &kvmv1.Eviction{ + ObjectMeta: metav1.ObjectMeta{Name: hypervisorName.Name}, + Spec: kvmv1.EvictionSpec{ + Hypervisor: hypervisorName.Name, + Reason: "test", + }, + } + Expect(controllerutil.SetControllerReference(hypervisor, eviction, controller.Scheme)).To(Succeed()) + Expect(k8sClient.Create(ctx, eviction)).To(Succeed()) + + meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeEvicting, + Status: metav1.ConditionFalse, + Message: "done", + Reason: kvmv1.ConditionReasonSucceeded, + }) + Expect(k8sClient.Status().Update(ctx, eviction)).To(Succeed()) + + // Set NumInstances=1 on the Hypervisor (simulating late-arriving instance) + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + hypervisor.Status.NumInstances = 1 + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeEvicting, + Status: metav1.ConditionFalse, + Reason: kvmv1.ConditionReasonSucceeded, + Message: "Evicted", + }) + hypervisor.Status.Evicted = true + Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) + }) + + It("should flip Evicted back to false", func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + Expect(hypervisor.Status.Evicted).To(BeFalse()) + }) + + It("should set Evicting back to Running", func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + cond := meta.FindStatusCondition(hypervisor.Status.Conditions, kvmv1.ConditionTypeEvicting) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(kvmv1.ConditionReasonRunning)) + }) + + It("should delete the eviction CR to re-enter drain", func(ctx SpecContext) { + eviction := &kvmv1.Eviction{} + err := k8sClient.Get(ctx, hypervisorName, eviction) + Expect(err).To(HaveOccurred()) + Expect(k8sclient.IgnoreNotFound(err)).To(Succeed()) + }) + }) + + Context("when eviction just finished but instances remain (no pre-existing Succeeded condition)", func() { + BeforeEach(func(ctx SpecContext) { + // Simulate: eviction CR exists and reports finished, but the + // Hypervisor does NOT have Evicting=Succeeded yet (first reconcile + // after the Eviction CR transitioned). + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + + eviction := &kvmv1.Eviction{ + ObjectMeta: metav1.ObjectMeta{Name: hypervisorName.Name}, + Spec: kvmv1.EvictionSpec{ + Hypervisor: hypervisorName.Name, + Reason: "test", + }, + } + Expect(controllerutil.SetControllerReference(hypervisor, eviction, controller.Scheme)).To(Succeed()) + Expect(k8sClient.Create(ctx, eviction)).To(Succeed()) + + meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeEvicting, + Status: metav1.ConditionFalse, + Message: "done", + Reason: kvmv1.ConditionReasonSucceeded, + }) + Expect(k8sClient.Status().Update(ctx, eviction)).To(Succeed()) + + // Set NumInstances=1 but leave Evicting condition as Running (not Succeeded) + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + hypervisor.Status.NumInstances = 1 + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeEvicting, + Status: metav1.ConditionTrue, + Reason: kvmv1.ConditionReasonRunning, + Message: "Evicting", + }) + Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) + }) + + It("should set Evicting to True/Running and not declare Evicted", func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + Expect(hypervisor.Status.Evicted).To(BeFalse()) + cond := meta.FindStatusCondition(hypervisor.Status.Conditions, kvmv1.ConditionTypeEvicting) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(kvmv1.ConditionReasonRunning)) + }) + + It("should delete the eviction CR to restart drain", func(ctx SpecContext) { + eviction := &kvmv1.Eviction{} + err := k8sClient.Get(ctx, hypervisorName, eviction) + Expect(err).To(HaveOccurred()) + Expect(k8sclient.IgnoreNotFound(err)).To(Succeed()) + }) + }) + }) }) var _ = Describe("retainStatusCondition", func() { diff --git a/internal/openstack/migrations.go b/internal/openstack/migrations.go new file mode 100644 index 00000000..9a92ee20 --- /dev/null +++ b/internal/openstack/migrations.go @@ -0,0 +1,173 @@ +/* +SPDX-FileCopyrightText: Copyright 2025 SAP SE or an SAP affiliate company and cobaltcore-dev contributors +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openstack + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/gophercloud/gophercloud/v2" +) + +// IncomingMigrationsLookbackWindow defines how far back we query Nova for +// migrations. Nova's live-migration monitor updates Migration.updated_at every +// ~5 seconds on both libvirt/kvm and libvirt/cloud-hypervisor backends, so any +// in-flight migration is guaranteed to appear in this window. +const IncomingMigrationsLookbackWindow = 6 * time.Hour + +// MigrationInfo holds the subset of Nova migration fields relevant to settling +// incoming migrations on a host being drained. +type MigrationInfo struct { + ID int `json:"id"` + UUID string `json:"uuid"` + InstanceUUID string `json:"instance_uuid"` + Status string `json:"status"` + SourceCompute string `json:"source_compute"` + DestCompute string `json:"dest_compute"` + MigrationType string `json:"migration_type"` +} + +// migrationsResponse mirrors Nova's GET /os-migrations JSON envelope. +type migrationsResponse struct { + Migrations []MigrationInfo `json:"migrations"` +} + +// terminalStatuses are migration statuses that no longer represent an in-flight +// migration. Derived from nova/db/main/api.py migration_get_in_progress_by_host_and_node. +var terminalStatuses = map[string]bool{ + "confirmed": true, + "reverted": true, + "error": true, + "failed": true, + "completed": true, + "cancelled": true, + "done": true, +} + +// abortableStatuses are migration statuses where a DELETE (abort) is supported. +// queued and preparing require microversion >= 2.65; running is always abortable. +// The operator uses microversion 2.90 which covers all three. +var abortableStatuses = map[string]bool{ + "queued": true, + "preparing": true, + "running": true, +} + +// ListActiveIncomingMigrations queries Nova for migrations that target the +// given host and are still in a non-terminal state. It issues a single call +// with a changes-since window of IncomingMigrationsLookbackWindow, then +// filters client-side for dest_compute==host and non-terminal status. +func ListActiveIncomingMigrations(ctx context.Context, sc *gophercloud.ServiceClient, host string) ([]MigrationInfo, error) { + changesSince := time.Now().UTC().Add(-IncomingMigrationsLookbackWindow).Format(time.RFC3339) + + migrations, err := listMigrationsForHost(ctx, sc, host, changesSince) + if err != nil { + return nil, fmt.Errorf("listing migrations for host %s: %w", host, err) + } + + var result []MigrationInfo + for _, m := range migrations { + if m.DestCompute != host { + continue + } + if terminalStatuses[m.Status] { + continue + } + result = append(result, m) + } + + return result, nil +} + +// AbortMigration issues DELETE /servers/{instanceUUID}/migrations/{migrationID} +// to abort a live-migration. Treats HTTP 404 and 409 as non-fatal (the +// migration already transitioned past an abortable state or was deleted). +func AbortMigration(ctx context.Context, sc *gophercloud.ServiceClient, instanceUUID string, migrationID int) error { + deleteURL := sc.ServiceURL("servers", instanceUUID, "migrations", strconv.Itoa(migrationID)) + + resp, err := sc.Delete(ctx, deleteURL, &gophercloud.RequestOpts{ + OkCodes: []int{http.StatusAccepted, http.StatusNoContent}, + }) + if err != nil { + if gophercloud.ResponseCodeIs(err, http.StatusNotFound) || gophercloud.ResponseCodeIs(err, http.StatusConflict) { + // Migration already completed/cancelled or not found — non-fatal. + return nil + } + return fmt.Errorf("aborting migration %d for instance %s: %w", migrationID, instanceUUID, err) + } + // gophercloud closes the response body when JSONResponse is nil, + // but close defensively if it wasn't. + if resp != nil && resp.Body != nil { + resp.Body.Close() + } + return nil +} + +// SettleIncomingMigrations combines listing and aborting: it lists all active +// incoming migrations for the host, aborts those in abortable states, and +// returns both the aborted and the waiting (post-migrating) sets. +// +// The caller should treat: +// - len(aborted) > 0 as "just issued aborts, recheck on next reconcile" +// - len(waiting) > 0 as "cannot abort, must wait for completion" +// - both empty as "no incoming migrations; host is settled" +func SettleIncomingMigrations(ctx context.Context, sc *gophercloud.ServiceClient, host string) (aborted, waiting []MigrationInfo, err error) { + active, err := ListActiveIncomingMigrations(ctx, sc, host) + if err != nil { + return nil, nil, err + } + + for _, m := range active { + if abortableStatuses[m.Status] && m.MigrationType == "live-migration" { + if abortErr := AbortMigration(ctx, sc, m.InstanceUUID, m.ID); abortErr != nil { + return aborted, waiting, abortErr + } + aborted = append(aborted, m) + } else { + // Non-abortable: either post-migrating, or an evacuation (which Nova + // does not support aborting). Must wait for completion. + waiting = append(waiting, m) + } + } + + return aborted, waiting, nil +} + +// listMigrationsForHost queries GET /os-migrations with host and changes-since filters. +func listMigrationsForHost(ctx context.Context, sc *gophercloud.ServiceClient, host, changesSince string) ([]MigrationInfo, error) { + query := url.Values{} + query.Set("host", host) + query.Set("changes-since", changesSince) + + requestURL := sc.ServiceURL("os-migrations") + "?" + query.Encode() + + var parsed migrationsResponse + //nolint:bodyclose // gophercloud closes the body when JSONResponse is non-nil + _, err := sc.Get(ctx, requestURL, &parsed, &gophercloud.RequestOpts{ + OkCodes: []int{http.StatusOK}, + }) + if err != nil { + return nil, err + } + + return parsed.Migrations, nil +} diff --git a/internal/openstack/migrations_test.go b/internal/openstack/migrations_test.go new file mode 100644 index 00000000..d22b916c --- /dev/null +++ b/internal/openstack/migrations_test.go @@ -0,0 +1,618 @@ +/* +SPDX-FileCopyrightText: Copyright 2025 SAP SE or an SAP affiliate company and cobaltcore-dev contributors +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openstack + +import ( + "context" + "fmt" + "net/http" + + "github.com/gophercloud/gophercloud/v2/testhelper" + "github.com/gophercloud/gophercloud/v2/testhelper/client" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Migrations", func() { + var ( + fakeServer testhelper.FakeServer + ctx context.Context + ) + + BeforeEach(func() { + fakeServer = testhelper.SetupHTTP() + ctx = context.Background() + }) + + AfterEach(func() { + fakeServer.Teardown() + }) + + // migrationsHandler returns a handler that responds to GET /os-migrations. + migrationsHandler := func(data string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, data) + } + } + + emptyMigrations := `{"migrations": []}` + + Describe("ListActiveIncomingMigrations", func() { + Context("when there are no migrations", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(emptyMigrations)) + }) + + It("should return an empty list", func() { + sc := client.ServiceClient(fakeServer) + migrations, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(migrations).To(BeEmpty()) + }) + }) + + Context("when there is a running migration targeting the host", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 42, + "uuid": "mig-uuid-1", + "instance_uuid": "inst-uuid-1", + "status": "running", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + } + ]}`)) + }) + + It("should return the migration", func() { + sc := client.ServiceClient(fakeServer) + migrations, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(migrations).To(HaveLen(1)) + Expect(migrations[0].ID).To(Equal(42)) + Expect(migrations[0].InstanceUUID).To(Equal("inst-uuid-1")) + Expect(migrations[0].Status).To(Equal("running")) + Expect(migrations[0].DestCompute).To(Equal("node009-bb549")) + }) + }) + + Context("when there is a post-migrating migration targeting the host", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 99, + "uuid": "mig-uuid-2", + "instance_uuid": "inst-uuid-2", + "status": "post-migrating", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + } + ]}`)) + }) + + It("should return the migration as active incoming", func() { + sc := client.ServiceClient(fakeServer) + migrations, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(migrations).To(HaveLen(1)) + Expect(migrations[0].Status).To(Equal("post-migrating")) + }) + }) + + Context("when there is an outbound migration (source_compute == host)", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 50, + "uuid": "mig-uuid-3", + "instance_uuid": "inst-uuid-3", + "status": "running", + "source_compute": "node009-bb549", + "dest_compute": "node005-bb549", + "migration_type": "live-migration" + } + ]}`)) + }) + + It("should ignore outbound migrations", func() { + sc := client.ServiceClient(fakeServer) + migrations, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(migrations).To(BeEmpty()) + }) + }) + + Context("when there is an incoming evacuation", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 77, + "uuid": "mig-uuid-4", + "instance_uuid": "inst-uuid-4", + "status": "running", + "source_compute": "node001-bb549", + "dest_compute": "node009-bb549", + "migration_type": "evacuation" + } + ]}`)) + }) + + It("should include incoming evacuations", func() { + sc := client.ServiceClient(fakeServer) + migrations, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(migrations).To(HaveLen(1)) + Expect(migrations[0].MigrationType).To(Equal("evacuation")) + Expect(migrations[0].DestCompute).To(Equal("node009-bb549")) + }) + }) + + Context("when migrations have terminal statuses", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 10, + "uuid": "mig-t1", + "instance_uuid": "inst-t1", + "status": "completed", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 11, + "uuid": "mig-t2", + "instance_uuid": "inst-t2", + "status": "cancelled", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 12, + "uuid": "mig-t3", + "instance_uuid": "inst-t3", + "status": "error", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 13, + "uuid": "mig-t4", + "instance_uuid": "inst-t4", + "status": "failed", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 14, + "uuid": "mig-t5", + "instance_uuid": "inst-t5", + "status": "confirmed", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 15, + "uuid": "mig-t6", + "instance_uuid": "inst-t6", + "status": "reverted", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 16, + "uuid": "mig-t7", + "instance_uuid": "inst-t7", + "status": "done", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + } + ]}`)) + }) + + It("should filter out all terminal statuses", func() { + sc := client.ServiceClient(fakeServer) + migrations, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(migrations).To(BeEmpty()) + }) + }) + + Context("when Nova returns a server error", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, `{"error": "Internal Server Error"}`) + }) + }) + + It("should propagate the error", func() { + sc := client.ServiceClient(fakeServer) + _, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).To(HaveOccurred()) + }) + }) + + Context("with a mixed set of migrations", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 1, + "uuid": "mig-1", + "instance_uuid": "inst-1", + "status": "running", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 2, + "uuid": "mig-2", + "instance_uuid": "inst-2", + "status": "running", + "source_compute": "node007-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 3, + "uuid": "mig-3", + "instance_uuid": "inst-3", + "status": "post-migrating", + "source_compute": "node002-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 4, + "uuid": "mig-4", + "instance_uuid": "inst-4", + "status": "running", + "source_compute": "node009-bb549", + "dest_compute": "node005-bb549", + "migration_type": "live-migration" + }, + { + "id": 5, + "uuid": "mig-5", + "instance_uuid": "inst-5", + "status": "completed", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + } + ]}`)) + }) + + It("should return only active incoming migrations", func() { + sc := client.ServiceClient(fakeServer) + migrations, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(migrations).To(HaveLen(3)) + ids := []int{migrations[0].ID, migrations[1].ID, migrations[2].ID} + Expect(ids).To(ConsistOf(1, 2, 3)) + }) + }) + + Context("verifies query parameters", func() { + var receivedQueries []string + + BeforeEach(func() { + receivedQueries = nil + fakeServer.Mux.HandleFunc("GET /os-migrations", func(w http.ResponseWriter, r *http.Request) { + receivedQueries = append(receivedQueries, r.URL.RawQuery) + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, `{"migrations": []}`) + }) + }) + + It("should send host and changes-since parameters", func() { + sc := client.ServiceClient(fakeServer) + _, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + // Single call without migration_type filter + Expect(receivedQueries).To(HaveLen(1)) + Expect(receivedQueries[0]).To(ContainSubstring("host=node009-bb549")) + Expect(receivedQueries[0]).To(ContainSubstring("changes-since=")) + Expect(receivedQueries[0]).NotTo(ContainSubstring("migration_type")) + }) + }) + }) + + Describe("AbortMigration", func() { + Context("when abort succeeds", func() { + var deleteCalled bool + + BeforeEach(func() { + deleteCalled = false + fakeServer.Mux.HandleFunc("DELETE /servers/inst-uuid-1/migrations/42", func(w http.ResponseWriter, r *http.Request) { + deleteCalled = true + w.WriteHeader(http.StatusAccepted) + }) + }) + + It("should issue DELETE and return nil", func() { + sc := client.ServiceClient(fakeServer) + err := AbortMigration(ctx, sc, "inst-uuid-1", 42) + Expect(err).NotTo(HaveOccurred()) + Expect(deleteCalled).To(BeTrue()) + }) + }) + + Context("when Nova returns 404 (migration already gone)", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("DELETE /servers/inst-uuid-1/migrations/42", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + fmt.Fprint(w, `{"itemNotFound": {"message": "Migration not found", "code": 404}}`) + }) + }) + + It("should treat as non-fatal", func() { + sc := client.ServiceClient(fakeServer) + err := AbortMigration(ctx, sc, "inst-uuid-1", 42) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("when Nova returns 409 (migration not abortable)", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("DELETE /servers/inst-uuid-1/migrations/42", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusConflict) + fmt.Fprint(w, `{"conflictingRequest": {"message": "Migration status is post-migrating", "code": 409}}`) + }) + }) + + It("should treat as non-fatal", func() { + sc := client.ServiceClient(fakeServer) + err := AbortMigration(ctx, sc, "inst-uuid-1", 42) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("when Nova returns a server error", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("DELETE /servers/inst-uuid-1/migrations/42", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, `{"error": "Internal Server Error"}`) + }) + }) + + It("should propagate the error", func() { + sc := client.ServiceClient(fakeServer) + err := AbortMigration(ctx, sc, "inst-uuid-1", 42) + Expect(err).To(HaveOccurred()) + }) + }) + }) + + Describe("SettleIncomingMigrations", func() { + Context("when there are no incoming migrations", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(emptyMigrations)) + }) + + It("should return empty aborted and waiting", func() { + sc := client.ServiceClient(fakeServer) + aborted, waiting, err := SettleIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(aborted).To(BeEmpty()) + Expect(waiting).To(BeEmpty()) + }) + }) + + Context("when there are abortable migrations", func() { + var deleteCalls int + + BeforeEach(func() { + deleteCalls = 0 + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 1, + "uuid": "mig-1", + "instance_uuid": "inst-1", + "status": "running", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 2, + "uuid": "mig-2", + "instance_uuid": "inst-2", + "status": "queued", + "source_compute": "node007-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + } + ]}`)) + + fakeServer.Mux.HandleFunc("DELETE /servers/inst-1/migrations/1", func(w http.ResponseWriter, r *http.Request) { + deleteCalls++ + w.WriteHeader(http.StatusAccepted) + }) + + fakeServer.Mux.HandleFunc("DELETE /servers/inst-2/migrations/2", func(w http.ResponseWriter, r *http.Request) { + deleteCalls++ + w.WriteHeader(http.StatusAccepted) + }) + }) + + It("should abort all and return them in aborted list", func() { + sc := client.ServiceClient(fakeServer) + aborted, waiting, err := SettleIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(aborted).To(HaveLen(2)) + Expect(waiting).To(BeEmpty()) + Expect(deleteCalls).To(Equal(2)) + }) + }) + + Context("when there is a post-migrating migration (not abortable)", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 99, + "uuid": "mig-99", + "instance_uuid": "inst-99", + "status": "post-migrating", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + } + ]}`)) + }) + + It("should return in waiting list without issuing DELETE", func() { + sc := client.ServiceClient(fakeServer) + aborted, waiting, err := SettleIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(aborted).To(BeEmpty()) + Expect(waiting).To(HaveLen(1)) + Expect(waiting[0].ID).To(Equal(99)) + }) + }) + + Context("with a mixed set (abortable + post-migrating + outbound + completed)", func() { + var deleteCalls int + + BeforeEach(func() { + deleteCalls = 0 + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 1, + "uuid": "mig-1", + "instance_uuid": "inst-1", + "status": "running", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 2, + "uuid": "mig-2", + "instance_uuid": "inst-2", + "status": "preparing", + "source_compute": "node007-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 3, + "uuid": "mig-3", + "instance_uuid": "inst-3", + "status": "post-migrating", + "source_compute": "node002-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 4, + "uuid": "mig-4", + "instance_uuid": "inst-4", + "status": "running", + "source_compute": "node009-bb549", + "dest_compute": "node005-bb549", + "migration_type": "live-migration" + }, + { + "id": 5, + "uuid": "mig-5", + "instance_uuid": "inst-5", + "status": "completed", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + } + ]}`)) + + fakeServer.Mux.HandleFunc("DELETE /servers/inst-1/migrations/1", func(w http.ResponseWriter, r *http.Request) { + deleteCalls++ + w.WriteHeader(http.StatusAccepted) + }) + + fakeServer.Mux.HandleFunc("DELETE /servers/inst-2/migrations/2", func(w http.ResponseWriter, r *http.Request) { + deleteCalls++ + w.WriteHeader(http.StatusAccepted) + }) + }) + + It("should abort 2, wait on 1, ignore outbound and completed", func() { + sc := client.ServiceClient(fakeServer) + aborted, waiting, err := SettleIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(aborted).To(HaveLen(2)) + Expect(waiting).To(HaveLen(1)) + Expect(waiting[0].ID).To(Equal(3)) + Expect(deleteCalls).To(Equal(2)) + }) + }) + + Context("when Nova list returns an error", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, `{"error": "Internal Server Error"}`) + }) + }) + + It("should propagate the error", func() { + sc := client.ServiceClient(fakeServer) + _, _, err := SettleIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).To(HaveOccurred()) + }) + }) + + Context("when there is a running evacuation (not abortable via migration-abort)", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 88, + "uuid": "mig-evac-1", + "instance_uuid": "inst-evac-1", + "status": "running", + "source_compute": "node001-bb549", + "dest_compute": "node009-bb549", + "migration_type": "evacuation" + } + ]}`)) + }) + + It("should place the evacuation in waiting without issuing DELETE", func() { + sc := client.ServiceClient(fakeServer) + aborted, waiting, err := SettleIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(aborted).To(BeEmpty()) + Expect(waiting).To(HaveLen(1)) + Expect(waiting[0].ID).To(Equal(88)) + Expect(waiting[0].MigrationType).To(Equal("evacuation")) + }) + }) + }) +})