[PATCH QEMU HCI-8.1 v5 30/36] VZ: migration/cpr-exec: take the vhost-net devices back if exec fails

[Andrey Drobyshev <andrey.drobyshev@virtuozzo.com>]

Right before execve(), cpr_exec_cb() releases the kernel vhost-net
devices with VHOST_RESET_OWNER, so that the new QEMU can take them.  If
execve() fails, the VM goes on in the old QEMU, but nobody takes the
devices back: the kernel refuses the ioctls of the next
vhost_net_start(), and the guest is left without network.

Take them back with VHOST_SET_OWNER when the exec fails, and set the
vring call and err fds again: VHOST_RESET_OWNER dropped them too, and
vhost_dev_init() registers them only once.  The devices are stopped at
that point, and vhost_net_start() sets up the rest again when the VM
resumes.  Only the devices that were actually released are taken back:
vhost_net_cpr_reset_owners() marks them.  vhost-vsock does the same in
its CPR notifier.

Signed-off-by: Andrey Drobyshev <andrey.drobyshev@virtuozzo.com>
---
 hw/net/vhost_net.c        | 49 +++++++++++++++++++++++++++++++++++++++++++++++
 include/hw/virtio/vhost.h |  2 ++
 include/net/vhost_net.h   |  1 +
 migration/cpr-exec.c      |  2 ++
 4 files changed, 54 insertions(+)

diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c
index 56c697f4bcd..fae6b060c6f 100644
--- a/hw/net/vhost_net.c
+++ b/hw/net/vhost_net.c
@@ -594,6 +594,55 @@ void vhost_net_cpr_reset_owners(void)
             if (r < 0 && r != -ENOSYS) {
                 warn_report("cpr: failed to reset vhost-net owner for %s: %s",
                             nc->name, strerror(-r));
+            } else if (r == 0) {
+                net->cpr_owner_reset = true;
+            }
+        }
+    }
+}
+
+/*
+ * Take the ownership back when the exec fails and the VM goes on in the
+ * old QEMU.  VHOST_RESET_OWNER has reset the kernel device, but the devices
+ * are stopped, and vhost_net_start() sets everything up again when the VM
+ * resumes.
+ */
+void vhost_net_cpr_set_owners(void)
+{
+    NetClientState *nc;
+
+    QTAILQ_FOREACH(nc, &net_clients, next) {
+        VHostNetState *net = get_vhost_net(nc);
+
+        if (net && net->cpr_owner_reset) {
+            struct vhost_dev *dev = &net->dev;
+            int i, r = vhost_dev_set_owner(dev);
+
+            if (r < 0) {
+                warn_report("cpr: failed to set vhost-net owner for %s: %s",
+                            nc->name, strerror(-r));
+                continue;
+            }
+            net->cpr_owner_reset = false;
+
+            /* RESET_OWNER also dropped the vring fds set by vhost_dev_init() */
+            for (i = 0; i < dev->nvqs; i++) {
+                const VhostOps *ops = dev->vhost_ops;
+                struct vhost_virtqueue *vq = &dev->vqs[i];
+                struct vhost_vring_file file = {
+                    .index = ops->vhost_get_vq_index(dev, dev->vq_index + i),
+                    .fd = event_notifier_get_wfd(&vq->masked_notifier),
+                };
+
+                r = ops->vhost_set_vring_call(dev, &file);
+                if (r == 0 && ops->vhost_set_vring_err) {
+                    file.fd = event_notifier_get_fd(&vq->error_notifier);
+                    r = ops->vhost_set_vring_err(dev, &file);
+                }
+                if (r < 0) {
+                    warn_report("cpr: failed to set vring fds of %s: %s",
+                                nc->name, strerror(-r));
+                }
             }
         }
     }
diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
index a92439d3443..421f7139cfb 100644
--- a/include/hw/virtio/vhost.h
+++ b/include/hw/virtio/vhost.h
@@ -151,6 +151,8 @@ struct vhost_net {
     SaveAcketFeatures *save_acked_features;
     bool is_vhost_user;
     NetClientState *nc;
+    /* released for cpr-exec, take back if it fails */
+    bool cpr_owner_reset;
 };
 
 /**
diff --git a/include/net/vhost_net.h b/include/net/vhost_net.h
index 39ce7ad0e3d..cdfefa81500 100644
--- a/include/net/vhost_net.h
+++ b/include/net/vhost_net.h
@@ -90,4 +90,5 @@ int vhost_net_virtqueue_restart(VirtIODevice *vdev, NetClientState *nc,
 void vhost_net_save_acked_features(NetClientState *nc);
 
 void vhost_net_cpr_reset_owners(void);
+void vhost_net_cpr_set_owners(void);
 #endif
diff --git a/migration/cpr-exec.c b/migration/cpr-exec.c
index df11b0c7633..e4ea160698c 100644
--- a/migration/cpr-exec.c
+++ b/migration/cpr-exec.c
@@ -253,6 +253,8 @@ fail:
     if (cpr_ram_rdonly) {
         cpr_ram_restore_rw();
     }
+    /* Still the same mm, take back the vhost-net devices we released */
+    vhost_net_cpr_set_owners();
 
     /* Note, we can go from state COMPLETED to FAILED */
     migration_call_notifiers(MIG_EVENT_PRECOPY_FAILED, NULL);

-- 
2.47.1