[PATCH QEMU HCI-8.1 16/21] vfio/migration: don't migrate device state during CPR

[Andrey Drobyshev <andrey.drobyshev@virtuozzo.com>]

cpr-transfer and cpr-exec hand the running VFIO device over to the new
QEMU as is: the device fd, its mappings and interrupts are preserved, and
the device itself is never stopped.

But if the device supports VFIO migration, its migration handlers still
run.  The old QEMU stops the device and saves its state, then the new
QEMU puts the very same device into RESUMING and loads that state back.
On top of that, vfio_pci_load_config() re-programs MSI-X and disables
vector 0.

For an NVIDIA vGPU this breaks the device after qemu-update: restoring
the state into the live vGPU fails and the guest loses the device.
Devices without migration support aren't affected, since no migration
handlers are registered for them.

Skip VFIO migration in CPR modes: don't save or load the device state,
and don't change the device state on VM stop/start.  Use
cpr_incoming_needed(), which tells exactly when the device is preserved
by CPR, both on the old and the new QEMU side.

Signed-off-by: Andrey Drobyshev <andrey.drobyshev@virtuozzo.com>
---
 hw/vfio/migration.c | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
index 4c06e3db936..99d285e7b20 100644
--- a/hw/vfio/migration.c
+++ b/hw/vfio/migration.c
@@ -19,6 +19,7 @@
 #include "hw/vfio/vfio-device.h"
 #include "hw/vfio/vfio-migration.h"
 #include "migration/misc.h"
+#include "migration/cpr.h"
 #include "migration/savevm.h"
 #include "migration/vmstate.h"
 #include "migration/qemu-file.h"
@@ -674,6 +675,16 @@ static void vfio_save_state(QEMUFile *f, void *opaque)
     Error *local_err = NULL;
     int ret;
 
+    /*
+     * .is_active doesn't cover the non-iterable stage, so the section is
+     * still sent: leave it empty, or new QEMU would load the config into
+     * the preserved device and re-program its MSI-X.
+     */
+    if (cpr_incoming_needed(NULL)) {
+        qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
+        return;
+    }
+
     if (vfio_multifd_transfer_enabled(vbasedev)) {
         if (vfio_load_config_after_iter(vbasedev)) {
             qemu_put_be64(f, VFIO_MIG_FLAG_DEV_CONFIG_LOAD_READY);
@@ -824,7 +835,19 @@ static int vfio_switchover_start(void *opaque)
     return 0;
 }
 
+/*
+ * cpr-transfer and cpr-exec hand the running device over to the new QEMU
+ * as is: its fds, mappings and interrupts are preserved and it never stops.
+ * Device state migration must stay out of it, else the device is stopped,
+ * its state saved and then resumed into the very same device.
+ */
+static bool vfio_is_active(void *opaque)
+{
+    return !cpr_incoming_needed(NULL);
+}
+
 static const SaveVMHandlers savevm_vfio_handlers = {
+    .is_active = vfio_is_active,
     .save_prepare = vfio_save_prepare,
     .save_setup = vfio_save_setup,
     .save_cleanup = vfio_save_cleanup,
@@ -857,6 +880,10 @@ static void vfio_vmstate_change_prepare(void *opaque, bool running,
     Error *local_err = NULL;
     int ret;
 
+    if (cpr_incoming_needed(NULL)) {
+        return;
+    }
+
     new_state = migration->device_state == VFIO_DEVICE_STATE_PRE_COPY ?
                     VFIO_DEVICE_STATE_PRE_COPY_P2P :
                     VFIO_DEVICE_STATE_RUNNING_P2P;
@@ -882,6 +909,10 @@ static void vfio_vmstate_change(void *opaque, bool running, RunState state)
     Error *local_err = NULL;
     int ret;
 
+    if (cpr_incoming_needed(NULL)) {
+        return;
+    }
+
     if (running) {
         new_state = VFIO_DEVICE_STATE_RUNNING;
     } else {

-- 
2.47.1