feat(reservations): steer CR VM live migrations toward reservation slot hosts #981
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,168 @@ | ||
| // Copyright SAP SE | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package weighers | ||
|
|
||
| import ( | ||
| "context" | ||
| "log/slog" | ||
|
|
||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
|
|
||
| api "github.com/cobaltcore-dev/cortex/api/external/nova" | ||
| "github.com/cobaltcore-dev/cortex/api/v1alpha1" | ||
| "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" | ||
| "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" | ||
| hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" | ||
| ) | ||
|
|
||
| // Options for the KVM CR migration slot weigher. | ||
| type KVMCRMigrationSlotOpts struct { | ||
| // Weight assigned to hosts that can accommodate the CR reservation slot. | ||
| // Default: 0.1 | ||
| SlotHostWeight *float64 `json:"slotHostWeight,omitempty"` | ||
| // Weight assigned to hosts that cannot accommodate the CR reservation slot. | ||
| // Default: 0.0 | ||
| DefaultHostWeight *float64 `json:"defaultHostWeight,omitempty"` | ||
| } | ||
|
|
||
| func (o KVMCRMigrationSlotOpts) Validate() error { | ||
| return nil | ||
| } | ||
|
|
||
| func (o KVMCRMigrationSlotOpts) GetSlotHostWeight() float64 { | ||
| if o.SlotHostWeight == nil { | ||
| return 0.1 | ||
| } | ||
| return *o.SlotHostWeight | ||
| } | ||
|
|
||
| func (o KVMCRMigrationSlotOpts) GetDefaultHostWeight() float64 { | ||
| if o.DefaultHostWeight == nil { | ||
| return 0.0 | ||
| } | ||
| return *o.DefaultHostWeight | ||
| } | ||
|
|
||
| // KVMCRMigrationSlotStep weighs live-migration candidates by whether they can | ||
| // accommodate the CR reservation slot of the migrating VM. | ||
| // | ||
| // When a VM with a CR reservation slot is migrated, this weigher boosts hosts | ||
| // that either have a ready CR reservation with sufficient remaining capacity for | ||
| // the slot, or have enough free memory for the slot to follow via the reconciler. | ||
| // This steers the migration toward hosts where double-blocking is avoided or | ||
| // minimised. | ||
| // | ||
| // If the VM has no confirmed CR reservation, all candidates receive equal weight | ||
| // (no effect on ranking). | ||
| // | ||
| // Only activates for LiveMigrationIntent. | ||
| type KVMCRMigrationSlotStep struct { | ||
| lib.BaseWeigher[api.ExternalSchedulerRequest, KVMCRMigrationSlotOpts] | ||
| } | ||
|
|
||
| func (s *KVMCRMigrationSlotStep) Run( | ||
| traceLog *slog.Logger, | ||
| request api.ExternalSchedulerRequest, | ||
| ) (*lib.FilterWeigherPipelineStepResult, error) { | ||
| result := s.IncludeAllHostsFromRequest(request) | ||
|
|
||
| intent, err := request.GetIntent() | ||
| if err != nil || intent != api.LiveMigrationIntent { | ||
| traceLog.Info("not a live migration, skipping CR migration slot weigher") | ||
| return result, nil //nolint:nilerr | ||
| } | ||
|
|
||
| instanceUUID := request.Spec.Data.InstanceUUID | ||
| projectID := request.Spec.Data.ProjectID | ||
|
|
||
| var allReservations v1alpha1.ReservationList | ||
| if err := s.Client.List(context.Background(), &allReservations, | ||
| client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource}, | ||
| ); err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| // Find the source slot that has this VM confirmed. | ||
| var sourceSlot *v1alpha1.Reservation | ||
| for i := range allReservations.Items { | ||
| res := &allReservations.Items[i] | ||
| if !res.IsReady() { | ||
| continue | ||
| } | ||
| if res.Status.CommittedResourceReservation == nil || res.Spec.CommittedResourceReservation == nil { | ||
| continue | ||
| } | ||
| if _, ok := res.Status.CommittedResourceReservation.Allocations[instanceUUID]; ok { | ||
| sourceSlot = res | ||
| break | ||
| } | ||
| } | ||
|
|
||
| if sourceSlot == nil { | ||
| traceLog.Info("migrating VM has no confirmed CR reservation slot, skipping slot weigher", | ||
| "instanceUUID", instanceUUID) | ||
| CRMigrationSlotMetricsSingleton.RecordResult("no_source_slot") | ||
| return result, nil | ||
| } | ||
|
|
||
| slotMemoryBytes := sourceSlot.Spec.Resources[hv1.ResourceMemory] | ||
| if slotMemoryBytes.IsZero() { | ||
| traceLog.Info("source CR slot has no memory resource, skipping slot weigher", | ||
| "instanceUUID", instanceUUID, | ||
| "reservation", sourceSlot.Name) | ||
| CRMigrationSlotMetricsSingleton.RecordResult("invalid_source_slot") | ||
| return result, nil | ||
| } | ||
|
|
||
| resourceGroup := sourceSlot.Spec.CommittedResourceReservation.ResourceGroup | ||
|
|
||
| traceLog.Info("found source CR reservation slot for migrating VM", | ||
| "instanceUUID", instanceUUID, | ||
| "reservation", sourceSlot.Name, | ||
| "slotMemoryBytes", slotMemoryBytes.Value(), | ||
| "resourceGroup", resourceGroup, | ||
| ) | ||
|
|
||
| evaluator, err := crs.BuildSlotEvaluatorFromReservations(context.Background(), s.Client, allReservations.Items) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| slotHostWeight := s.Options.GetSlotHostWeight() | ||
| defaultHostWeight := s.Options.GetDefaultHostWeight() | ||
|
|
||
| slotFound := false | ||
| for host := range result.Activations { | ||
| hasSlot := evaluator.HasSlotWithCapacity(host, projectID, resourceGroup, slotMemoryBytes.Value()) | ||
| canFit := evaluator.CanAccommodateSlot(host, slotMemoryBytes.Value()) | ||
| if hasSlot { | ||
|
Check failure on line 139 in internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go
|
||
| result.Activations[host] = slotHostWeight | ||
| slotFound = true | ||
| traceLog.Info("host has existing CR slot for migration, boosting weight", | ||
| "host", host, "weight", slotHostWeight) | ||
| } else if canFit { | ||
| result.Activations[host] = slotHostWeight | ||
| traceLog.Info("host can accommodate slot via reconciler, boosting weight", | ||
| "host", host, "weight", slotHostWeight) | ||
| } else { | ||
| result.Activations[host] = defaultHostWeight | ||
| traceLog.Info("host cannot accommodate CR slot, applying low weight", | ||
| "host", host, "weight", defaultHostWeight) | ||
| } | ||
| } | ||
|
|
||
| if slotFound { | ||
| CRMigrationSlotMetricsSingleton.RecordResult("slot_found") | ||
| } else { | ||
| CRMigrationSlotMetricsSingleton.RecordResult("no_slot") | ||
| } | ||
|
|
||
| return result, nil | ||
| } | ||
|
|
||
| func init() { | ||
| Index["kvm_cr_migration_slot"] = func() NovaWeigher { | ||
| return &KVMCRMigrationSlotStep{} | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| // Copyright SAP SE | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package weighers | ||
|
|
||
| import ( | ||
| "log/slog" | ||
| "sync" | ||
|
|
||
| "github.com/prometheus/client_golang/prometheus" | ||
| ) | ||
|
|
||
| // CRMigrationSlotMetrics holds Prometheus metrics for the CR migration slot weigher. | ||
| type CRMigrationSlotMetrics struct { | ||
| // Results counts live migration requests by outcome: | ||
| // - slot_found: at least one candidate has a compatible CR slot | ||
| // - no_slot: source slot found but no candidate is compatible | ||
| // - no_source_slot: migrating VM has no confirmed CR reservation | ||
| // - invalid_source_slot: source slot found but has no memory resource | ||
| Results *prometheus.CounterVec | ||
| } | ||
|
|
||
| func NewCRMigrationSlotMetrics() *CRMigrationSlotMetrics { | ||
| return &CRMigrationSlotMetrics{ | ||
| Results: prometheus.NewCounterVec( | ||
| prometheus.CounterOpts{ | ||
| Name: "cortex_nova_weigh_cr_migration_slot_requests_total", | ||
| Help: "Live migration requests processed by the CR migration slot weigher, " + | ||
| "labeled by outcome (slot_found, no_slot, no_source_slot).", | ||
| }, | ||
| []string{"result"}, | ||
| ), | ||
| } | ||
| } | ||
|
|
||
| func (m *CRMigrationSlotMetrics) Describe(ch chan<- *prometheus.Desc) { | ||
| if m == nil || m.Results == nil { | ||
| return | ||
| } | ||
| m.Results.Describe(ch) | ||
| } | ||
|
|
||
| func (m *CRMigrationSlotMetrics) Collect(ch chan<- prometheus.Metric) { | ||
| if m == nil || m.Results == nil { | ||
| return | ||
| } | ||
| m.Results.Collect(ch) | ||
| } | ||
|
|
||
| var recordCRMigrationSlotResultNilOnce = &sync.Once{} | ||
|
|
||
| func (m *CRMigrationSlotMetrics) RecordResult(result string) { | ||
| if m == nil || m.Results == nil { | ||
| recordCRMigrationSlotResultNilOnce.Do(func() { | ||
| slog.Warn("CRMigrationSlotMetrics is nil; result metric not recorded "+ | ||
| "(is CRMigrationSlotMetricsSingleton initialized in cmd/manager?)", | ||
| "result", result, | ||
| ) | ||
| }) | ||
| return | ||
| } | ||
| m.Results.WithLabelValues(result).Inc() | ||
| } | ||
|
|
||
| // CRMigrationSlotMetricsSingleton is set from cmd/manager/main.go during initialization. | ||
| var CRMigrationSlotMetricsSingleton *CRMigrationSlotMetrics |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.