Skip to content

Commit

Permalink
Merge pull request #11330 from jean-edouard/miginfo
Browse files Browse the repository at this point in the history 
migration state: add source pod and failure reason
  • Loading branch information
@kubevirt-bot
kubevirt-bot committed Feb 26, 2024
2 parents bdca916 + 563d33d commit 22aa13d
Show file tree
Hide file tree
Showing 12 changed files with 100 additions and 13 deletions.
7 changes: 7 additions & 0 deletions api/openapi-spec/swagger.json
View file
Expand Up @@ -20100,6 +20100,10 @@
"description": "Indicates that the migration failed",
"type": "boolean"
},
"failureReason": {
"description": "Contains the reason why the migration failed",
"type": "string"
},
"migrationConfiguration": {
"description": "Migration configurations to apply",
"$ref": "#/definitions/v1.MigrationConfiguration"
Expand All @@ -20120,6 +20124,9 @@
"description": "The source node that the VMI originated on",
"type": "string"
},
"sourcePod": {
"type": "string"
},
"startTimestamp": {
"description": "The time the migration action began",
"$ref": "#/definitions/k8s.io.apimachinery.pkg.apis.meta.v1.Time"
Expand Down
17 changes: 17 additions & 0 deletions pkg/controller/controller.go
View file
Expand Up @@ -346,6 +346,23 @@ func SetVMIMigrationPhaseTransitionTimestamp(oldVMIMigration *v1.VirtualMachineI
}
}

func SetSourcePod(migration *v1.VirtualMachineInstanceMigration, vmi *v1.VirtualMachineInstance, podInformer cache.SharedIndexInformer) {
if migration.Status.Phase != v1.MigrationPending {
return
}
sourcePod, err := CurrentVMIPod(vmi, podInformer)
if err != nil {
log.Log.Object(vmi).Reason(err).Warning("migration source pod not found")
}
if sourcePod != nil {
if migration.Status.MigrationState == nil {
migration.Status.MigrationState = &v1.VirtualMachineInstanceMigrationState{}
}
migration.Status.MigrationState.SourcePod = sourcePod.Name
}

}

func VMIHasHotplugVolumes(vmi *v1.VirtualMachineInstance) bool {
for _, volumeStatus := range vmi.Status.VolumeStatus {
if volumeStatus.HotplugVolume != nil {
Expand Down
12 changes: 10 additions & 2 deletions pkg/virt-controller/watch/migration.go
View file
Expand Up @@ -505,13 +505,13 @@ func (c *MigrationController) updateStatus(migration *virtv1.VirtualMachineInsta
}

controller.SetVMIMigrationPhaseTransitionTimestamp(migration, migrationCopy)
controller.SetSourcePod(migrationCopy, vmi, c.podInformer)

if !equality.Semantic.DeepEqual(migration.Status, migrationCopy.Status) {
err := c.statusUpdater.UpdateStatus(migrationCopy)
if err != nil {
return err
}

} else if !equality.Semantic.DeepEqual(migration.Finalizers, migrationCopy.Finalizers) {
_, err := c.clientset.VirtualMachineInstanceMigration(migrationCopy.Namespace).Update(migrationCopy)
if err != nil {
Expand Down Expand Up @@ -837,7 +837,12 @@ func (c *MigrationController) handleMarkMigrationFailedOnVMI(migration *virtv1.V
return err
}
log.Log.Object(vmi).Infof("Marked Migration %s/%s failed on vmi due to target pod disappearing before migration kicked off.", migration.Namespace, migration.Name)
c.recorder.Event(vmi, k8sv1.EventTypeWarning, FailedMigrationReason, fmt.Sprintf("VirtualMachineInstance migration uid %s failed. reason: target pod is down", string(migration.UID)))
failureReason := "Target pod is down"
c.recorder.Event(vmi, k8sv1.EventTypeWarning, FailedMigrationReason, fmt.Sprintf("VirtualMachineInstance migration uid %s failed. reason: %s", string(migration.UID), failureReason))
if vmiCopy.Status.MigrationState.FailureReason == "" {
// Only set the failure reason if empty, as virt-handler may already have provided a better one
vmiCopy.Status.MigrationState.FailureReason = failureReason
}

return nil
}
Expand Down Expand Up @@ -875,6 +880,9 @@ func (c *MigrationController) handleTargetPodHandoff(migration *virtv1.VirtualMa
SourceNode: vmi.Status.NodeName,
TargetPod: pod.Name,
}
if migration.Status.MigrationState != nil {
vmiCopy.Status.MigrationState.SourcePod = migration.Status.MigrationState.SourcePod
}

// By setting this label, virt-handler on the target node will receive
// the vmi and prepare the local environment for the migration
Expand Down
19 changes: 19 additions & 0 deletions pkg/virt-controller/watch/migration_test.go
View file
Expand Up @@ -444,6 +444,7 @@ var _ = Describe("Migration watcher", func() {
vmi.Status.SelinuxContext = "system_u:system_r:container_file_t:s0:c1,c2"
addVirtualMachineInstance(vmi)
podFeeder.Add(targetPod)
shouldExpectGenericMigrationUpdate()
shouldExpectAttachmentPodCreation(vmi.UID, migration.UID)

controller.Execute()
Expand Down Expand Up @@ -633,6 +634,7 @@ var _ = Describe("Migration watcher", func() {
return vmiReturn, nil
})

shouldExpectGenericMigrationUpdate()
shouldExpectPodCreation(vmi.UID, migration.UID, 1, 0, 0)
controller.Execute()
testutils.ExpectEvents(recorder, SuccessfulCreatePodReason)
Expand All @@ -643,6 +645,7 @@ var _ = Describe("Migration watcher", func() {

addMigration(migration)
addVirtualMachineInstance(vmi)
shouldExpectGenericMigrationUpdate()
shouldExpectPodCreation(vmi.UID, migration.UID, 1, 0, 0)

controller.Execute()
Expand All @@ -661,6 +664,7 @@ var _ = Describe("Migration watcher", func() {

addMigration(migration)
addVirtualMachineInstance(vmi)
shouldExpectGenericMigrationUpdate()

controller.Execute()
})
Expand Down Expand Up @@ -693,6 +697,7 @@ var _ = Describe("Migration watcher", func() {
addVirtualMachineInstance(vmi)
}

shouldExpectGenericMigrationUpdate()
shouldExpectPodCreation(vmi.UID, migration.UID, 1, 0, 0)
controller.Execute()
testutils.ExpectEvent(recorder, SuccessfulCreatePodReason)
Expand All @@ -716,6 +721,8 @@ var _ = Describe("Migration watcher", func() {
addVirtualMachineInstance(vmi)
}

shouldExpectGenericMigrationUpdate()

controller.Execute()
})

Expand Down Expand Up @@ -749,6 +756,8 @@ var _ = Describe("Migration watcher", func() {
Expect(podInformer.GetStore().Add(pod)).To(Succeed())
}

shouldExpectGenericMigrationUpdate()

controller.Execute()
})

Expand All @@ -769,6 +778,7 @@ var _ = Describe("Migration watcher", func() {
addVirtualMachineInstance(vmi)
}

shouldExpectGenericMigrationUpdate()
shouldExpectPodCreation(vmi.UID, migration.UID, 1, 0, 0)
controller.Execute()
testutils.ExpectEvent(recorder, SuccessfulCreatePodReason)
Expand All @@ -791,6 +801,7 @@ var _ = Describe("Migration watcher", func() {
addVirtualMachineInstance(vmi)
}

shouldExpectGenericMigrationUpdate()
controller.Execute()
})

Expand Down Expand Up @@ -845,6 +856,7 @@ var _ = Describe("Migration watcher", func() {

addMigration(migration)
addVirtualMachineInstance(vmi)
shouldExpectGenericMigrationUpdate()
shouldExpectPodCreation(vmi.UID, migration.UID, 2, 1, 1)

controller.Execute()
Expand Down Expand Up @@ -1550,6 +1562,8 @@ var _ = Describe("Migration watcher", func() {
addVirtualMachineInstance(vmi)
addNode(node)

shouldExpectGenericMigrationUpdate()

expectPodToHaveProperNodeSelector := func(pod *k8sv1.Pod) {
podHasCpuModeLabelSelector := false
for key, _ := range pod.Spec.NodeSelector {
Expand Down Expand Up @@ -1589,6 +1603,7 @@ var _ = Describe("Migration watcher", func() {
addVirtualMachineInstance(vmi)
addPDB(pdb)

shouldExpectGenericMigrationUpdate()
shouldExpectPDBPatch(vmi, migration)
controller.Execute()

Expand All @@ -1611,6 +1626,7 @@ var _ = Describe("Migration watcher", func() {
addVirtualMachineInstance(vmi)
addPDB(pdb)

shouldExpectGenericMigrationUpdate()
shouldExpectPodCreation(vmi.UID, migration.UID, 1, 0, 0)

controller.Execute()
Expand All @@ -1631,6 +1647,7 @@ var _ = Describe("Migration watcher", func() {

addMigration(migration)
addVirtualMachineInstance(vmi)
shouldExpectGenericMigrationUpdate()
addPDB(pdb)

shouldExpectPDBPatch(vmi, migration)
Expand Down Expand Up @@ -2016,6 +2033,7 @@ var _ = Describe("Migration watcher", func() {

addMigration(migration)
addVirtualMachineInstance(vmi)
shouldExpectGenericMigrationUpdate()
shouldExpectTargetPodWithSELinuxLevel("s0:c1,c2")

controller.Execute()
Expand All @@ -2034,6 +2052,7 @@ var _ = Describe("Migration watcher", func() {

addMigration(migration)
addVirtualMachineInstance(vmi)
shouldExpectGenericMigrationUpdate()
shouldExpectTargetPodWithSELinuxLevel("")

controller.Execute()
Expand Down
5 changes: 1 addition & 4 deletions pkg/virt-controller/watch/vmi.go
View file
Expand Up @@ -1136,10 +1136,7 @@ func isPodFailed(pod *k8sv1.Pod) bool {
}

func podExists(pod *k8sv1.Pod) bool {
if pod != nil {
return true
}
return false
return pod != nil
}

func (c *VMIController) hotplugPodsReady(vmi *virtv1.VirtualMachineInstance, virtLauncherPod *k8sv1.Pod) (bool, syncError) {
Expand Down
2 changes: 1 addition & 1 deletion pkg/virt-handler/vm.go
View file
Expand Up @@ -594,7 +594,6 @@ func canUpdateToUnmounted(currentPhase v1.VolumePhase) bool {
}

func (d *VirtualMachineController) setMigrationProgressStatus(vmi *v1.VirtualMachineInstance, domain *api.Domain) {

if domain == nil ||
domain.Spec.Metadata.KubeVirt.Migration == nil ||
vmi.Status.MigrationState == nil ||
Expand All @@ -609,6 +608,7 @@ func (d *VirtualMachineController) setMigrationProgressStatus(vmi *v1.VirtualMac

if vmi.Status.MigrationState.EndTimestamp == nil && migrationMetadata.EndTimestamp != nil {
if migrationMetadata.Failed {
vmi.Status.MigrationState.FailureReason = migrationMetadata.FailureReason
d.recorder.Event(vmi, k8sv1.EventTypeWarning, v1.Migrated.String(), fmt.Sprintf("VirtualMachineInstance migration uid %s failed. reason:%s", string(migrationMetadata.UID), migrationMetadata.FailureReason))
}
}
Expand Down
10 changes: 10 additions & 0 deletions pkg/virt-operator/resource/generate/components/validations_generated.go
View file
Expand Up @@ -12196,6 +12196,9 @@ var CRDsValidation map[string]string = map[string]string{
failed:
description: Indicates that the migration failed
type: boolean
failureReason:
description: Contains the reason why the migration failed
type: string
migrationConfiguration:
description: Migration configurations to apply
properties:
Expand Down Expand Up @@ -12292,6 +12295,8 @@ var CRDsValidation map[string]string = map[string]string{
sourceNode:
description: The source node that the VMI originated on
type: string
sourcePod:
type: string
startTimestamp:
description: The time the migration action began
format: date-time
Expand Down Expand Up @@ -12594,6 +12599,9 @@ var CRDsValidation map[string]string = map[string]string{
failed:
description: Indicates that the migration failed
type: boolean
failureReason:
description: Contains the reason why the migration failed
type: string
migrationConfiguration:
description: Migration configurations to apply
properties:
Expand Down Expand Up @@ -12690,6 +12698,8 @@ var CRDsValidation map[string]string = map[string]string{
sourceNode:
description: The source node that the VMI originated on
type: string
sourcePod:
type: string
startTimestamp:
description: The time the migration action began
format: date-time
Expand Down
5 changes: 5 additions & 0 deletions staging/src/kubevirt.io/api/core/v1/types.go
View file
Expand Up @@ -708,8 +708,11 @@ type VirtualMachineInstanceMigrationState struct {
TargetPod string `json:"targetPod,omitempty"`
// The UID of the target attachment pod for hotplug volumes
TargetAttachmentPodUID types.UID `json:"targetAttachmentPodUID,omitempty"`

// The source node that the VMI originated on
SourceNode string `json:"sourceNode,omitempty"`
SourcePod string `json:"sourcePod,omitempty"`

// Indicates the migration completed
Completed bool `json:"completed,omitempty"`
// Indicates that the migration failed
Expand All @@ -718,6 +721,8 @@ type VirtualMachineInstanceMigrationState struct {
AbortRequested bool `json:"abortRequested,omitempty"`
// Indicates the final status of the live migration abortion
AbortStatus MigrationAbortStatus `json:"abortStatus,omitempty"`
// Contains the reason why the migration failed
FailureReason string `json:"failureReason,omitempty"`
// The VirtualMachineInstanceMigration object associated with this migration
MigrationUID types.UID `json:"migrationUid,omitempty"`
// Lets us know if the vmi is currently running pre or post copy migration
Expand Down
1 change: 1 addition & 0 deletions staging/src/kubevirt.io/api/core/v1/types_swagger_generated.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 13 additions & 0 deletions staging/src/kubevirt.io/client-go/api/openapi_generated.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion tests/libmigration/migration.go
View file
Expand Up @@ -357,7 +357,7 @@ func CreateNodeAffinityRuleToMigrateFromSourceToTargetAndBack(sourceNode *k8sv1.
},
}, nil
}
func ConfirmVMIPostMigrationFailed(vmi *v1.VirtualMachineInstance, migrationUID string) {
func ConfirmVMIPostMigrationFailed(vmi *v1.VirtualMachineInstance, migrationUID string) *v1.VirtualMachineInstance {
virtClient := kubevirt.Client()
By("Retrieving the VMI post migration")
vmi, err := virtClient.VirtualMachineInstance(vmi.Namespace).Get(context.Background(), vmi.Name, &metav1.GetOptions{})
Expand All @@ -376,6 +376,8 @@ func ConfirmVMIPostMigrationFailed(vmi *v1.VirtualMachineInstance, migrationUID

By("Verifying the VMI's is in the running state")
Expect(vmi.Status.Phase).To(Equal(v1.Running))

return vmi
}

func ConfirmVMIPostMigrationAborted(vmi *v1.VirtualMachineInstance, migrationUID string, timeout int) *v1.VirtualMachineInstance {
Expand Down

0 comments on commit 22aa13d

Please sign in to comment.