[PATCH QEMU HCI-8.1 v3 21/28] vfio/migration: don't migrate device state during CPR

[Andrey Drobyshev <andrey.drobyshev@virtuozzo.com>]

cpr-transfer and cpr-exec hand the running VFIO device over to the new
QEMU as is: the device fd, its mappings and interrupts are preserved, and
the device itself is never stopped.

But if the device supports VFIO migration, its migration handlers still
run.  The old QEMU stops the device and saves its state, then the new
QEMU puts the very same device into RESUMING and loads that state back.
On top of that, vfio_pci_load_config() re-programs MSI-X and disables
vector 0.

For an NVIDIA vGPU this breaks the device after qemu-update: restoring
the state into the live vGPU fails and the guest loses the device.
Devices without migration support aren't affected, since no migration
handlers are registered for them.

Skip VFIO migration while a CPR migration is in progress: don't save or
load the device state, and don't change the device state on VM
stop/start.  All of it is gated on cpr_is_in_progress(), not on the mode
alone: the mode may merely be set, and then a plain stop/cont must still
change the device state, and a snapshot must still save it.

Also reset multifd_transfer in vfio_multifd_cleanup(): it's only set in
vfio_multifd_setup(), which CPR now skips, so a stale value from an
earlier migration would still let the multifd save thread run.

Signed-off-by: Andrey Drobyshev <andrey.drobyshev@virtuozzo.com>
---
 hw/vfio/migration-multifd.c |  2 ++
 hw/vfio/migration.c         | 38 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/hw/vfio/migration-multifd.c b/hw/vfio/migration-multifd.c
index e4785031a73..452f772dd7a 100644
--- a/hw/vfio/migration-multifd.c
+++ b/hw/vfio/migration-multifd.c
@@ -567,6 +567,8 @@ void vfio_multifd_cleanup(VFIODevice *vbasedev)
     VFIOMigration *migration = vbasedev->migration;
 
     g_clear_pointer(&migration->multifd, vfio_multifd_free);
+    /* Decided anew by vfio_multifd_setup() of the next migration */
+    migration->multifd_transfer = false;
 }
 
 bool vfio_multifd_transfer_supported(void)
diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
index 4c06e3db936..b142c9f2289 100644
--- a/hw/vfio/migration.c
+++ b/hw/vfio/migration.c
@@ -19,6 +19,7 @@
 #include "hw/vfio/vfio-device.h"
 #include "hw/vfio/vfio-migration.h"
 #include "migration/misc.h"
+#include "migration/cpr.h"
 #include "migration/savevm.h"
 #include "migration/vmstate.h"
 #include "migration/qemu-file.h"
@@ -513,7 +514,6 @@ static void vfio_save_cleanup(void *opaque)
     Error *local_err = NULL;
     int ret;
 
-    /* Currently a NOP, done for symmetry with load_cleanup() */
     vfio_multifd_cleanup(vbasedev);
 
     /*
@@ -674,6 +674,16 @@ static void vfio_save_state(QEMUFile *f, void *opaque)
     Error *local_err = NULL;
     int ret;
 
+    /*
+     * .is_active doesn't cover the non-iterable stage, so the section is
+     * still sent: leave it empty, or new QEMU would load the config into
+     * the preserved device and re-program its MSI-X.
+     */
+    if (cpr_is_in_progress()) {
+        qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
+        return;
+    }
+
     if (vfio_multifd_transfer_enabled(vbasedev)) {
         if (vfio_load_config_after_iter(vbasedev)) {
             qemu_put_be64(f, VFIO_MIG_FLAG_DEV_CONFIG_LOAD_READY);
@@ -810,6 +820,11 @@ static bool vfio_switchover_ack_needed(void *opaque)
 {
     VFIODevice *vbasedev = opaque;
 
+    /* No device state is sent during CPR, so no ACK to wait for */
+    if (cpr_is_in_progress()) {
+        return false;
+    }
+
     return vfio_precopy_supported(vbasedev);
 }
 
@@ -824,7 +839,19 @@ static int vfio_switchover_start(void *opaque)
     return 0;
 }
 
+/*
+ * cpr-transfer and cpr-exec hand the running device over to the new QEMU
+ * as is: its fds, mappings and interrupts are preserved and it never stops.
+ * Device state migration must stay out of it, else the device is stopped,
+ * its state saved and then resumed into the very same device.
+ */
+static bool vfio_is_active(void *opaque)
+{
+    return !cpr_is_in_progress();
+}
+
 static const SaveVMHandlers savevm_vfio_handlers = {
+    .is_active = vfio_is_active,
     .save_prepare = vfio_save_prepare,
     .save_setup = vfio_save_setup,
     .save_cleanup = vfio_save_cleanup,
@@ -857,6 +884,11 @@ static void vfio_vmstate_change_prepare(void *opaque, bool running,
     Error *local_err = NULL;
     int ret;
 
+    /* The device is preserved, not a plain stop/cont: keep its state */
+    if (cpr_is_in_progress()) {
+        return;
+    }
+
     new_state = migration->device_state == VFIO_DEVICE_STATE_PRE_COPY ?
                     VFIO_DEVICE_STATE_PRE_COPY_P2P :
                     VFIO_DEVICE_STATE_RUNNING_P2P;
@@ -882,6 +914,10 @@ static void vfio_vmstate_change(void *opaque, bool running, RunState state)
     Error *local_err = NULL;
     int ret;
 
+    if (cpr_is_in_progress()) {
+        return;
+    }
+
     if (running) {
         new_state = VFIO_DEVICE_STATE_RUNNING;
     } else {

-- 
2.47.1