[PATCH QEMU HCI-8.1 20/21] VZ: vfio/region: preserve BAR mappings across cpr-exec #VSTOR-103860

[Andrey Drobyshev <andrey.drobyshev@virtuozzo.com>]

After cpr-exec, new QEMU mmaps the device BARs again on the preserved
device fd.  Some drivers don't allow that: the NVIDIA vGPU VFIO driver
refuses to map a VF BAR that is still recorded as mapped by old QEMU,
and that record is dropped only when the device fd is closed.  So BAR1
and BAR3 of the vGPU can't be mapped, and the guest loses the device.

Keep the mappings instead.  Save each BAR mapping in the CPR ramblock
list, so cpr_exec_cb() marks it with MADV_DOEXEC together with guest RAM,
and new QEMU finds it there and reuses it instead of calling mmap().
Same find-or-create pattern as for other preserved objects, e.g. vector
eventfds in vfio_notifier_init().

The ID is what cpr_name() would give to the mmap subregion:
"<device path>/<region name> mmaps[<index>]".

This needs a kernel which supports MADV_DOEXEC on shared PFN mappings of
device memory.  Without it cpr-exec of a VM with VFIO devices fails, and
the VM keeps running in old QEMU.

Signed-off-by: Andrey Drobyshev <andrey.drobyshev@virtuozzo.com>
---
 hw/vfio/region.c | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/hw/vfio/region.c b/hw/vfio/region.c
index 1deaef12fd0..07027ee79f3 100644
--- a/hw/vfio/region.c
+++ b/hw/vfio/region.c
@@ -30,6 +30,7 @@
 #include "qemu/units.h"
 #include "monitor/monitor.h"
 #include "system/ramblock.h"
+#include "migration/cpr.h"
 #include "vfio-helpers.h"
 
 /*
@@ -227,8 +228,39 @@ int vfio_region_setup(Object *obj, VFIODevice *vbasedev, VFIORegion *region,
     return 0;
 }
 
+static char *vfio_region_cpr_name(VFIORegion *region, int index)
+{
+    g_autofree char *path = qdev_get_dev_path(region->vbasedev->dev);
+    return g_strdup_printf("%s%s%s mmaps[%d]", path ?: "", path ? "/" : "",
+                           memory_region_name(region->mem), index);
+}
+
+static void *vfio_region_cpr_find(VFIORegion *region, int index)
+{
+    g_autofree char *name = vfio_region_cpr_name(region, index);
+    cpr_ramblock_t ram = cpr_find_ramblock(name);
+
+    if (ram.addr) {
+        assert(ram.length == region->mmaps[index].size);
+    }
+
+    return (void *)(uintptr_t)ram.addr;
+}
+
+static void vfio_region_cpr_save(VFIORegion *region, int index)
+{
+    g_autofree char *name = vfio_region_cpr_name(region, index);
+    void *addr = region->mmaps[index].mmap;
+    size_t size = region->mmaps[index].size;
+
+    cpr_save_ramblock(name, (cpr_ramblock_t) { (uintptr_t)addr, size });
+}
+
 static void vfio_subregion_unmap(VFIORegion *region, int index)
 {
+    g_autofree char *cpr_name = vfio_region_cpr_name(region, index);
+
+    cpr_delete_ramblock(cpr_name);
     trace_vfio_region_unmap(memory_region_name(&region->mmaps[index].mem),
                             region->mmaps[index].offset,
                             region->mmaps[index].offset +
@@ -314,6 +346,11 @@ int vfio_region_mmap(VFIORegion *region)
         size_t align = MIN(1ULL << ctz64(region->mmaps[i].size), 1 * GiB);
         void *map_base, *map_align;
 
+        region->mmaps[i].mmap = vfio_region_cpr_find(region, i);
+        if (region->mmaps[i].mmap) {
+            goto mapped;
+        }
+
         /*
          * Align the mmap for more efficient mapping in the kernel.  Ideally
          * we'd know the PMD and PUD mapping sizes to use as discrete alignment
@@ -347,7 +384,9 @@ int vfio_region_mmap(VFIORegion *region)
             ret = -errno;
             goto no_mmap;
         }
+        vfio_region_cpr_save(region, i);
 
+mapped:
         name = g_strdup_printf("%s mmaps[%d]",
                                memory_region_name(region->mem), i);
         memory_region_init_ram_device_ptr(&region->mmaps[i].mem,

-- 
2.47.1