[PATCH hci-dev v6 1/2] vfio/migration: stop a device left running after a guest shutdown #VSTOR-153976

[Andrey Drobyshev <andrey.drobyshev@virtuozzo.com>]

With -no-shutdown, a guest shutdown leaves the VM stopped. From there
QEMU can only reset the VM, quit, or migrate or save it. A quit or a
signal brings no VM state change, and neither does the switchover of a
migration from a stopped VM. So a device that is still running at that
point is closed or saved running.

Stop it first: in a shutdown notifier when QEMU quits, and at the start
of a save. The P2P round of the VM stop has already run, so the devices
are stopped one by one. A reset of the stopped VM leaves the device in
RUNNING_P2P. Nothing else does, so a device in that state is stopped in
any run state. Today only a failed migration of such a VM leaves a
device running there. The next patch does it at every guest shutdown.

This matters for NVIDIA vGPU (595.71.03): a VF closed while running,
after the Linux guest driver unloaded, breaks the next VM that resumes
on it. Its GPU stops answering.

Signed-off-by: Andrey Drobyshev <andrey.drobyshev@virtuozzo.com>
---
 hw/vfio/migration.c               | 47 +++++++++++++++++++++++++++++++
 hw/vfio/vfio-migration-internal.h |  1 +
 include/system/runstate.h         |  1 +
 system/runstate.c                 |  5 ++++
 4 files changed, 54 insertions(+)

diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
index 7e7fb496bbe..9c0ba2bde5c 100644
--- a/hw/vfio/migration.c
+++ b/hw/vfio/migration.c
@@ -415,6 +415,28 @@ static bool vfio_precopy_supported(VFIODevice *vbasedev)
     return migration->mig_flags & VFIO_MIGRATION_PRE_COPY;
 }
 
+/*
+ * Stop a device that a guest shutdown left running, before it is closed or
+ * saved. Only that leaves a device in RUNNING_P2P, or in RUNNING while the
+ * VM is in SHUTDOWN. The devices are stopped one by one: a device in
+ * RUNNING_P2P has been through the P2P round of the VM stop, a device in
+ * RUNNING was set there by a failed migration, which has no such round.
+ */
+static int vfio_migration_stop_after_shutdown(VFIODevice *vbasedev,
+                                              Error **errp)
+{
+    VFIOMigration *migration = vbasedev->migration;
+
+    if (migration->device_state != VFIO_DEVICE_STATE_RUNNING_P2P &&
+        !(runstate_check(RUN_STATE_SHUTDOWN) &&
+          migration->device_state == VFIO_DEVICE_STATE_RUNNING)) {
+        return 0;
+    }
+
+    return vfio_migration_set_state_or_reset(vbasedev, VFIO_DEVICE_STATE_STOP,
+                                             errp);
+}
+
 /* ---------------------------------------------------------------------- */
 
 static int vfio_save_prepare(void *opaque, Error **errp)
@@ -473,6 +495,12 @@ static int vfio_save_setup(QEMUFile *f, void *opaque, Error **errp)
     migration->event_save_iterate_started = false;
     migration->event_precopy_empty_hit = false;
 
+    /* A stopped VM gets no VM state change at switchover */
+    ret = vfio_migration_stop_after_shutdown(vbasedev, errp);
+    if (ret) {
+        return ret;
+    }
+
     if (vfio_precopy_supported(vbasedev)) {
         switch (migration->device_state) {
         case VFIO_DEVICE_STATE_RUNNING:
@@ -959,6 +987,22 @@ static void vfio_vmstate_change(void *opaque, bool running, RunState state)
                               mig_state_to_str(new_state));
 }
 
+static void vfio_migration_shutdown_notifier(Notifier *notifier, void *data)
+{
+    VFIOMigration *migration = container_of(notifier, VFIOMigration,
+                                            shutdown);
+    Error *local_err = NULL;
+
+    /* QEMU is quitting: a device left running must be stopped first */
+    if (qemu_shutdown_action_is_pause()) {
+        return;
+    }
+
+    if (vfio_migration_stop_after_shutdown(migration->vbasedev, &local_err)) {
+        error_report_err(local_err);
+    }
+}
+
 static int vfio_migration_state_notifier(NotifierWithReturn *notifier,
                                          MigrationEvent *e, Error **errp)
 {
@@ -1083,6 +1127,8 @@ static int vfio_migration_init(VFIODevice *vbasedev)
         vbasedev->dev, vfio_vmstate_change, prepare_cb, NULL, vbasedev);
     migration_add_notifier(&migration->migration_state,
                            vfio_migration_state_notifier);
+    migration->shutdown.notify = vfio_migration_shutdown_notifier;
+    qemu_register_shutdown_notifier(&migration->shutdown);
 
     return 0;
 }
@@ -1150,6 +1196,7 @@ static void vfio_migration_deinit(VFIODevice *vbasedev)
 {
     VFIOMigration *migration = vbasedev->migration;
 
+    notifier_remove(&migration->shutdown);
     migration_remove_notifier(&migration->migration_state);
     qemu_del_vm_change_state_handler(migration->vm_state);
     unregister_savevm(VMSTATE_IF(vbasedev->dev), "vfio", vbasedev);
diff --git a/hw/vfio/vfio-migration-internal.h b/hw/vfio/vfio-migration-internal.h
index 814fbd9ebae..e81d854d3c4 100644
--- a/hw/vfio/vfio-migration-internal.h
+++ b/hw/vfio/vfio-migration-internal.h
@@ -40,6 +40,7 @@ typedef struct VFIOMigration {
     struct VFIODevice *vbasedev;
     VMChangeStateEntry *vm_state;
     NotifierWithReturn migration_state;
+    Notifier shutdown;
     uint32_t device_state;
     int data_fd;
     void *data_buffer;
diff --git a/include/system/runstate.h b/include/system/runstate.h
index 929379adae4..29a919348da 100644
--- a/include/system/runstate.h
+++ b/include/system/runstate.h
@@ -148,6 +148,7 @@ void qemu_system_vmstop_request_prepare(void);
 bool qemu_vmstop_requested(RunState *r);
 ShutdownCause qemu_shutdown_requested_get(void);
 bool qemu_force_shutdown_requested(void);
+bool qemu_shutdown_action_is_pause(void);
 ShutdownCause qemu_reset_requested_get(void);
 void qemu_system_killed(int signal, pid_t pid);
 void qemu_system_reset(ShutdownCause reason);
diff --git a/system/runstate.c b/system/runstate.c
index 6178b0091a3..1cdecde0fcf 100644
--- a/system/runstate.c
+++ b/system/runstate.c
@@ -433,6 +433,11 @@ bool qemu_force_shutdown_requested(void)
     return force_shutdown;
 }
 
+bool qemu_shutdown_action_is_pause(void)
+{
+    return shutdown_action == SHUTDOWN_ACTION_PAUSE;
+}
+
 ShutdownCause qemu_reset_requested_get(void)
 {
     return reset_requested;
-- 
2.47.1