[PATCH QEMU HCI-8.0 v3 08/12] usb-host: hand off the device across cpr-transfer #VSTOR-137800

[Andrey Drobyshev <andrey.drobyshev@virtuozzo.com>]

Unlike cpr-exec, cpr-transfer keeps the source QEMU alive: it hands the
usbfs FD to the new QEMU over the migration channel and only exits once
the target is up.  While the handoff is in flight both processes hold the
same FD, and only one may reap URB events on it - if both do, one reaps
the other's in-flight completions and wedges an isochronous transfer that
cannot be retried.  So reaping is handed off along with the FD:

  * The target doesn't reap until it takes over: it stops events right
    after opening the FD in .realize() and restarts in .post_load().
  * The source stops reaping in .pre_save() and, since it stays alive,
    restarts from the MIG_EVENT_PRECOPY_FAILED notifier on failure.
  * On success the source's .exit() leaves the device untouched - no
    reset, interface release or host-driver rebind - and just closes its
    own copy of the FD.

Signed-off-by: Andrey Drobyshev <andrey.drobyshev@virtuozzo.com>
---
 hw/usb/host-libusb.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 83 insertions(+), 11 deletions(-)

diff --git a/hw/usb/host-libusb.c b/hw/usb/host-libusb.c
index 922c1a5585e..e29e6210fdd 100644
--- a/hw/usb/host-libusb.c
+++ b/hw/usb/host-libusb.c
@@ -110,6 +110,7 @@ struct USBHostDevice {
     NotifierWithReturn               cpr_notifier;
     GSList                           *cpr_inflight;
     bool                             cpr_drained;
+    bool                             cpr_handed_off;
     libusb_device                    *dev;
     libusb_device_handle             *dh;
     struct libusb_device_descriptor  ddesc;
@@ -1227,16 +1228,19 @@ static int usb_host_close(USBHostDevice *s)
 
     trace_usb_host_close(s->bus_num, s->addr);
 
-    usb_host_abort_xfers(s);
-    usb_host_iso_free_all(s);
+    if (!s->cpr_handed_off) {
+        usb_host_abort_xfers(s);
+        usb_host_iso_free_all(s);
 
-    if (udev->attached) {
-        usb_device_detach(udev);
+        if (udev->attached) {
+            usb_device_detach(udev);
+        }
+
+        usb_host_release_interfaces(s);
+        libusb_reset_device(s->dh);
+        usb_host_attach_kernel(s);
     }
 
-    usb_host_release_interfaces(s);
-    libusb_reset_device(s->dh);
-    usb_host_attach_kernel(s);
     libusb_close(s->dh);
     s->dh = NULL;
     s->dev = NULL;
@@ -1271,10 +1275,17 @@ static void usb_host_exit_notifier(struct Notifier *n, void *data)
     USBHostDevice *s = container_of(n, USBHostDevice, exit);
 
     if (s->dh) {
-        usb_host_abort_xfers(s);
-        usb_host_release_interfaces(s);
-        libusb_reset_device(s->dh);
-        usb_host_attach_kernel(s);
+        /*
+         * Handed to new QEMU across cpr-transfer: do not reset, release
+         * interfaces or rebind the host driver underneath it; just close
+         * our handle.
+         */
+        if (!s->cpr_handed_off) {
+            usb_host_abort_xfers(s);
+            usb_host_release_interfaces(s);
+            libusb_reset_device(s->dh);
+            usb_host_attach_kernel(s);
+        }
         libusb_close(s->dh);
     }
 }
@@ -1298,6 +1309,8 @@ static libusb_device *usb_host_find_ref(int bus, int addr)
 }
 
 static void usb_host_handle_data(USBDevice *udev, USBPacket *p);
+static void usb_host_cpr_stop_events(void);
+static void usb_host_cpr_restart_events(void);
 
 /*
  * A failed CPR migration resumes the source VM with the drained
@@ -1318,6 +1331,11 @@ static int usb_host_cpr_notifier(NotifierWithReturn *notifier,
         return 0;
     }
 
+    if (s->cpr_handed_off) {
+        usb_host_cpr_restart_events();
+        s->cpr_handed_off = false;
+    }
+
     for (it = s->cpr_inflight; it; it = it->next) {
         p = it->data;
         /* Replay from the start of the TD */
@@ -1390,6 +1408,14 @@ static void usb_host_realize(USBDevice *udev, Error **errp)
             error_setg(errp, "failed to open host usb device %s", s->hostdevice);
             return;
         }
+        if (cpr_is_incoming()) {
+            /*
+             * The source still owns the shared fd until it hands off.  Don't
+             * reap URBs on it yet, or we would steal the source's in-flight
+             * completions; .post_load() restarts events once we take over.
+             */
+            usb_host_cpr_stop_events();
+        }
     } else
 #endif
     if (s->match.addr && s->match.bus_num &&
@@ -1941,6 +1967,7 @@ static int usb_host_post_load(void *opaque, int version_id)
      * Skip it for CPR.
      */
     if (cpr_is_incoming()) {
+        usb_host_cpr_restart_events();
         return 0;
     }
 
@@ -1953,6 +1980,47 @@ static int usb_host_post_load(void *opaque, int version_id)
     return 0;
 }
 
+#ifndef CONFIG_WIN32
+
+/*
+ * cpr-transfer: the usbfs fd is shared with the new QEMU via SCM_RIGHTS.
+ * Stop this (source) process from reaping URBs on it, so the new QEMU can
+ * drive the device without both processes racing on the same fd.  Restart
+ * on migration failure, when the source resumes.
+ */
+static void usb_host_cpr_stop_events(void)
+{
+    const struct libusb_pollfd **poll = libusb_get_pollfds(ctx);
+
+    libusb_set_pollfd_notifiers(ctx, NULL, NULL, NULL);
+    if (poll) {
+        for (int i = 0; poll[i] != NULL; i++) {
+            usb_host_del_fd(poll[i]->fd, ctx);
+        }
+        free(poll);
+    }
+}
+
+static void usb_host_cpr_restart_events(void)
+{
+    const struct libusb_pollfd **poll = libusb_get_pollfds(ctx);
+
+    libusb_set_pollfd_notifiers(ctx, usb_host_add_fd, usb_host_del_fd, ctx);
+    if (poll) {
+        for (int i = 0; poll[i] != NULL; i++) {
+            usb_host_add_fd(poll[i]->fd, poll[i]->events, ctx);
+        }
+        free(poll);
+    }
+}
+
+#else
+
+static void usb_host_cpr_stop_events(void) {}
+static void usb_host_cpr_restart_events(void) {}
+
+#endif
+
 static int usb_host_pre_save(void *opaque)
 {
     USBHostDevice *s = opaque;
@@ -1964,6 +2032,10 @@ static int usb_host_pre_save(void *opaque)
         if (usb_host_cpr_drain_xfers(s) < 0) {
             return -1;
         }
+        if (mode == MIG_MODE_CPR_TRANSFER) {
+            usb_host_cpr_stop_events();
+            s->cpr_handed_off = true;
+        }
     }
     return 0;
 }

-- 
2.47.1