[PATCH QEMU HCI-8.0 v8 10/14] usb-host: hand off the device across cpr-transfer #VSTOR-137800

[Andrey Drobyshev <andrey.drobyshev@virtuozzo.com>]

Unlike cpr-exec, cpr-transfer keeps the source QEMU alive: it hands the
usbfs FD to the new QEMU over the migration channel and only exits once
the target is up.  While the handoff is in flight both processes hold the
same FD, and only one may reap URB events on it - if both do, one reaps
the other's in-flight completions and wedges an isochronous transfer that
cannot be retried.  So reaping is handed off along with the FD:

  * The target doesn't reap until it takes over: it stops events right
    after opening the FD in .realize() and restarts in .post_load().
    Until then it must not touch the device at all: if the incoming
    migration fails, the target exits while the source resumes, so the
    target's exit must not reset the device or release the interfaces
    underneath the source.
  * The source stops reaping when it quiesces the device on the
    stop-for-CPR transition (or in the .pre_save() fallback for an
    already-paused VM) and, since it stays alive, restarts reaping from
    the vm-state handler's resume branch on failure.
  * On success the source's .exit() leaves the device untouched - no
    reset, interface release or host-driver rebind - and just closes its
    own copy of the FD.

Reaping is stopped and restarted on the whole libusb context, which is
shared by all usb-host devices, so cpr-transfer currently supports a
single hostdevice= usb-host at a time.

Signed-off-by: Andrey Drobyshev <andrey.drobyshev@virtuozzo.com>
---
 hw/usb/host-libusb.c | 91 +++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 79 insertions(+), 12 deletions(-)

diff --git a/hw/usb/host-libusb.c b/hw/usb/host-libusb.c
index 70ae0a38167..1412f92c12e 100644
--- a/hw/usb/host-libusb.c
+++ b/hw/usb/host-libusb.c
@@ -127,6 +127,7 @@ struct USBHostDevice {
     GSList                           *cpr_inflight;
     bool                             cpr_drained;
     bool                             cpr_replay_all;
+    bool                             cpr_handed_off;
 
     /* request queues */
     QTAILQ_HEAD(, USBHostRequest)    requests;
@@ -283,6 +284,23 @@ static void usb_host_start_events(void)
     }
 }
 
+/*
+ * cpr-transfer shares the usbfs fd with the new QEMU, so stop reaping URBs on
+ * it here -- only one process may drive the device.
+ */
+static void usb_host_stop_events(void)
+{
+    const struct libusb_pollfd **poll = libusb_get_pollfds(ctx);
+
+    libusb_set_pollfd_notifiers(ctx, NULL, NULL, NULL);
+    if (poll) {
+        for (int i = 0; poll[i] != NULL; i++) {
+            usb_host_del_fd(poll[i]->fd, ctx);
+        }
+        free(poll);
+    }
+}
+
 #else
 
 static QEMUTimer *poll_timer;
@@ -306,6 +324,9 @@ static void usb_host_timer(void *opaque)
     usb_host_timer_kick();
 }
 
+static void usb_host_start_events(void) {}
+static void usb_host_stop_events(void) {}
+
 #endif /* !CONFIG_WIN32 */
 
 static int usb_host_init(void)
@@ -1285,13 +1306,18 @@ static int usb_host_cpr_drain_xfers(USBHostDevice *s)
 /*
  * Quiesce the URB data plane around a CPR switchover: drain the in-flight
  * transfers before the device/irqchip snapshot, so completions raised during
- * the drain are migrated too.
+ * the drain are migrated too.  For cpr-transfer, also hand reaping of the
+ * shared fd off to the target by stopping events on this side.
  */
 static int usb_host_cpr_quiesce(USBHostDevice *s, MigMode mode)
 {
     if (usb_host_cpr_drain_xfers(s) < 0) {
         return -1;
     }
+    if (mode == MIG_MODE_CPR_TRANSFER) {
+        usb_host_stop_events();
+        s->cpr_handed_off = true;
+    }
     return 0;
 }
 
@@ -1310,6 +1336,11 @@ static void usb_host_cpr_reissue(USBHostDevice *s)
         return;
     }
 
+    if (s->cpr_handed_off) {
+        usb_host_start_events();
+        s->cpr_handed_off = false;
+    }
+
     for (it = s->cpr_inflight; it; it = it->next) {
         p = it->data;
         /*
@@ -1384,16 +1415,19 @@ static int usb_host_close(USBHostDevice *s)
 
     trace_usb_host_close(s->bus_num, s->addr);
 
-    usb_host_abort_xfers(s);
-    usb_host_iso_free_all(s);
+    if (!s->cpr_handed_off) {
+        usb_host_abort_xfers(s);
+        usb_host_iso_free_all(s);
 
-    if (udev->attached) {
-        usb_device_detach(udev);
+        if (udev->attached) {
+            usb_device_detach(udev);
+        }
+
+        usb_host_release_interfaces(s);
+        libusb_reset_device(s->dh);
+        usb_host_attach_kernel(s);
     }
 
-    usb_host_release_interfaces(s);
-    libusb_reset_device(s->dh);
-    usb_host_attach_kernel(s);
     libusb_close(s->dh);
     s->dh = NULL;
     s->dev = NULL;
@@ -1428,10 +1462,17 @@ static void usb_host_exit_notifier(struct Notifier *n, void *data)
     USBHostDevice *s = container_of(n, USBHostDevice, exit);
 
     if (s->dh) {
-        usb_host_abort_xfers(s);
-        usb_host_release_interfaces(s);
-        libusb_reset_device(s->dh);
-        usb_host_attach_kernel(s);
+        /*
+         * Handed to new QEMU across cpr-transfer: do not reset, release
+         * interfaces or rebind the host driver underneath it; just close
+         * our handle.
+         */
+        if (!s->cpr_handed_off) {
+            usb_host_abort_xfers(s);
+            usb_host_release_interfaces(s);
+            libusb_reset_device(s->dh);
+            usb_host_attach_kernel(s);
+        }
         libusb_close(s->dh);
     }
 }
@@ -1500,6 +1541,18 @@ static void usb_host_realize(USBDevice *udev, Error **errp)
             error_setg(errp, "failed to open host usb device %s", s->hostdevice);
             return;
         }
+        if (cpr_get_incoming_mode() == MIG_MODE_CPR_TRANSFER) {
+            /*
+             * The source still owns the shared fd until it hands off: don't
+             * reap URBs on it yet, or we would steal the source's in-flight
+             * completions, and don't reset or release the device on an early
+             * exit either -- if the incoming migration fails, the source
+             * resumes with it.  .post_load() restarts events and takes
+             * ownership once the source has handed the fd off.
+             */
+            usb_host_stop_events();
+            s->cpr_handed_off = true;
+        }
     } else
 #endif
     if (s->match.addr && s->match.bus_num &&
@@ -1563,6 +1616,17 @@ static void usb_host_unrealize(USBDevice *udev)
     if (s->needs_autoscan) {
         QTAILQ_REMOVE(&hostdevs, s, next);
     }
+    /*
+     * An explicit device_del means we still own the device (a successful
+     * cpr-transfer handoff exits the process instead): restart the reaping
+     * stopped by the handoff and clear the flag to force a full
+     * reset/release in usb_host_close().  Except on a CPR target that
+     * hasn't taken over yet -- there the source still drives the fd.
+     */
+    if (s->cpr_handed_off && !cpr_is_incoming()) {
+        usb_host_start_events();
+        s->cpr_handed_off = false;
+    }
     usb_host_close(s);
     migrate_del_blocker(&s->cpr_blocker);
     if (s->cpr_vmse) {
@@ -2046,6 +2110,9 @@ static int usb_host_post_load(void *opaque, int version_id)
      * rescan in usb_host_post_load_bh() -- those are for cross-host migration.
      */
     if (cpr_is_incoming()) {
+        usb_host_start_events();
+        /* the takeover is complete: from here on this process owns the fd */
+        dev->cpr_handed_off = false;
         return 0;
     }
 

-- 
2.47.1