[PATCH QEMU HCI-8.0 v6 08/12] usb-host: hand off the device across cpr-transfer #VSTOR-137800

[Andrey Drobyshev <andrey.drobyshev@virtuozzo.com>]

Unlike cpr-exec, cpr-transfer keeps the source QEMU alive: it hands the
usbfs FD to the new QEMU over the migration channel and only exits once
the target is up.  While the handoff is in flight both processes hold the
same FD, and only one may reap URB events on it - if both do, one reaps
the other's in-flight completions and wedges an isochronous transfer that
cannot be retried.  So reaping is handed off along with the FD:

  * The target doesn't reap until it takes over: it stops events right
    after opening the FD in .realize() and restarts in .post_load().
  * The source stops reaping in .pre_save() and, since it stays alive,
    restarts reaping from the vm-state handler's resume branch on failure.
  * On success the source's .exit() leaves the device untouched - no
    reset, interface release or host-driver rebind - and just closes its
    own copy of the FD.

Signed-off-by: Andrey Drobyshev <andrey.drobyshev@virtuozzo.com>
---
 hw/usb/host-libusb.c | 109 +++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 96 insertions(+), 13 deletions(-)

diff --git a/hw/usb/host-libusb.c b/hw/usb/host-libusb.c
index 020a685aa41..492f0442974 100644
--- a/hw/usb/host-libusb.c
+++ b/hw/usb/host-libusb.c
@@ -127,6 +127,7 @@ struct USBHostDevice {
     GSList                           *cpr_inflight;
     bool                             cpr_drained;
     bool                             cpr_replay_all;
+    bool                             cpr_handed_off;
 
     /* request queues */
     QTAILQ_HEAD(, USBHostRequest)    requests;
@@ -1260,6 +1261,23 @@ static int usb_host_cpr_drain_xfers(USBHostDevice *s)
 }
 
 static void usb_host_cpr_reissue(USBHostDevice *s);
+static void usb_host_cpr_stop_events(void);
+
+/*
+ * Drain the device for a CPR switchover and, for cpr-transfer, hand reaping of
+ * the shared fd off to the target.  Returns -1 if the drain couldn't complete.
+ */
+static int usb_host_cpr_quiesce(USBHostDevice *s, MigMode mode)
+{
+    if (usb_host_cpr_drain_xfers(s) < 0) {
+        return -1;
+    }
+    if (mode == MIG_MODE_CPR_TRANSFER) {
+        usb_host_cpr_stop_events();
+        s->cpr_handed_off = true;
+    }
+    return 0;
+}
 
 /*
  * Quiesce the URB data plane around a CPR switchover: drain when the VM stops
@@ -1285,7 +1303,7 @@ static void usb_host_cpr_vm_state(void *opaque, bool running, RunState state)
     if (mode != MIG_MODE_CPR_EXEC && mode != MIG_MODE_CPR_TRANSFER) {
         return;
     }
-    if (usb_host_cpr_drain_xfers(s) < 0) {
+    if (usb_host_cpr_quiesce(s, mode) < 0) {
         migration_file_set_error(-EIO, NULL);
     }
 }
@@ -1300,16 +1318,19 @@ static int usb_host_close(USBHostDevice *s)
 
     trace_usb_host_close(s->bus_num, s->addr);
 
-    usb_host_abort_xfers(s);
-    usb_host_iso_free_all(s);
+    if (!s->cpr_handed_off) {
+        usb_host_abort_xfers(s);
+        usb_host_iso_free_all(s);
 
-    if (udev->attached) {
-        usb_device_detach(udev);
+        if (udev->attached) {
+            usb_device_detach(udev);
+        }
+
+        usb_host_release_interfaces(s);
+        libusb_reset_device(s->dh);
+        usb_host_attach_kernel(s);
     }
 
-    usb_host_release_interfaces(s);
-    libusb_reset_device(s->dh);
-    usb_host_attach_kernel(s);
     libusb_close(s->dh);
     s->dh = NULL;
     s->dev = NULL;
@@ -1344,10 +1365,17 @@ static void usb_host_exit_notifier(struct Notifier *n, void *data)
     USBHostDevice *s = container_of(n, USBHostDevice, exit);
 
     if (s->dh) {
-        usb_host_abort_xfers(s);
-        usb_host_release_interfaces(s);
-        libusb_reset_device(s->dh);
-        usb_host_attach_kernel(s);
+        /*
+         * Handed to new QEMU across cpr-transfer: do not reset, release
+         * interfaces or rebind the host driver underneath it; just close
+         * our handle.
+         */
+        if (!s->cpr_handed_off) {
+            usb_host_abort_xfers(s);
+            usb_host_release_interfaces(s);
+            libusb_reset_device(s->dh);
+            usb_host_attach_kernel(s);
+        }
         libusb_close(s->dh);
     }
 }
@@ -1371,6 +1399,7 @@ static libusb_device *usb_host_find_ref(int bus, int addr)
 }
 
 static void usb_host_handle_data(USBDevice *udev, USBPacket *p);
+static void usb_host_cpr_restart_events(void);
 
 /*
  * If CPR is aborted the source resumes with the drained packets still owned
@@ -1387,6 +1416,11 @@ static void usb_host_cpr_reissue(USBHostDevice *s)
         return;
     }
 
+    if (s->cpr_handed_off) {
+        usb_host_cpr_restart_events();
+        s->cpr_handed_off = false;
+    }
+
     for (it = s->cpr_inflight; it; it = it->next) {
         p = it->data;
         /* Replay from the start of the TD */
@@ -1455,6 +1489,14 @@ static void usb_host_realize(USBDevice *udev, Error **errp)
             error_setg(errp, "failed to open host usb device %s", s->hostdevice);
             return;
         }
+        if (cpr_is_incoming()) {
+            /*
+             * The source still owns the shared fd until it hands off.  Don't
+             * reap URBs on it yet, or we would steal the source's in-flight
+             * completions; .post_load() restarts events once we take over.
+             */
+            usb_host_cpr_stop_events();
+        }
     } else
 #endif
     if (s->match.addr && s->match.bus_num &&
@@ -2000,6 +2042,7 @@ static int usb_host_post_load(void *opaque, int version_id)
      * rescan in usb_host_post_load_bh() -- those are for cross-host migration.
      */
     if (cpr_is_incoming()) {
+        usb_host_cpr_restart_events();
         return 0;
     }
 
@@ -2012,6 +2055,46 @@ static int usb_host_post_load(void *opaque, int version_id)
     return 0;
 }
 
+#ifndef CONFIG_WIN32
+
+/*
+ * cpr-transfer shares the usbfs fd with the new QEMU, so stop reaping URBs on
+ * it here -- only one process may drive the device.  Restart if migration
+ * fails and the source resumes.
+ */
+static void usb_host_cpr_stop_events(void)
+{
+    const struct libusb_pollfd **poll = libusb_get_pollfds(ctx);
+
+    libusb_set_pollfd_notifiers(ctx, NULL, NULL, NULL);
+    if (poll) {
+        for (int i = 0; poll[i] != NULL; i++) {
+            usb_host_del_fd(poll[i]->fd, ctx);
+        }
+        free(poll);
+    }
+}
+
+static void usb_host_cpr_restart_events(void)
+{
+    const struct libusb_pollfd **poll = libusb_get_pollfds(ctx);
+
+    libusb_set_pollfd_notifiers(ctx, usb_host_add_fd, usb_host_del_fd, ctx);
+    if (poll) {
+        for (int i = 0; poll[i] != NULL; i++) {
+            usb_host_add_fd(poll[i]->fd, poll[i]->events, ctx);
+        }
+        free(poll);
+    }
+}
+
+#else
+
+static void usb_host_cpr_stop_events(void) {}
+static void usb_host_cpr_restart_events(void) {}
+
+#endif
+
 static int usb_host_pre_save(void *opaque)
 {
     USBHostDevice *s = opaque;
@@ -2025,7 +2108,7 @@ static int usb_host_pre_save(void *opaque)
     if ((mode == MIG_MODE_CPR_EXEC || mode == MIG_MODE_CPR_TRANSFER) &&
         s->dh && !s->cpr_drained) {
         s->cpr_replay_all = true;
-        if (usb_host_cpr_drain_xfers(s) < 0) {
+        if (usb_host_cpr_quiesce(s, mode) < 0) {
             return -1;
         }
     }

-- 
2.47.1