[Devel] [PATCH hci-8.1 1/3] fs/fuse kio: teach pcs_umem about hugepage-backed mappings

[Liu Kui <kui.liu@virtuozzo.com>]

pcs_umem pins every mapping as a flat array of 4k pages and the krpc
bvec builder emits one bio_vec per page, so a 1M chunk costs 256 page
pointers and 256 bio_vecs even when it sits in a single hugepage.

Detect hugepage backing after pinning: if the range is 2M-aligned and
each 2M chunk is backed by one folio (hugetlbfs or fully-mapped THP),
keep only the folio head page per chunk and raise umem->page_shift to
PMD_SHIFT. The bvec builder then caps bv_len at the backing unit
instead of PAGE_SIZE, so a 1M chunk in a hugepage umem becomes a
single bio_vec. Mixed or misaligned mappings stay at 4k.

Also pin with pin_user_pages_fast(), which needs no mmap_lock, and
fix the pcs_umem_get() error path, which leaked chunk page arrays and
could deref a NULL page_chunk.

https://virtuozzo.atlassian.net/browse/VSTOR-142166

Signed-off-by: Liu Kui <kui.liu@virtuozzo.com>
---
 fs/fuse/kio/pcs/pcs_krpc.c |  9 ++++-
 fs/fuse/kio/pcs/pcs_mr.c   | 84 +++++++++++++++++++++++++++++++++++++---------
 fs/fuse/kio/pcs/pcs_mr.h   | 45 +++++++++++++++++++++----
 3 files changed, 115 insertions(+), 23 deletions(-)

diff --git a/fs/fuse/kio/pcs/pcs_krpc.c b/fs/fuse/kio/pcs/pcs_krpc.c
index 7d40e17ce8ae..d4baf28cef71 100644
--- a/fs/fuse/kio/pcs/pcs_krpc.c
+++ b/fs/fuse/kio/pcs/pcs_krpc.c
@@ -410,6 +410,7 @@ static void kreq_prepare_data_buff(struct krpc_req *kreq, struct krpc_chunk *chu
 		}
 	} else {
 		struct pcs_umem *umem;
+		unsigned int contig;
 
 		umem = (chunk->type == KRPC_CHUNK_TYPE_MR) ? chunk->mr->umem : chunk->umem;
 
@@ -426,7 +427,13 @@ static void kreq_prepare_data_buff(struct krpc_req *kreq, struct krpc_chunk *chu
 			bvec->bv_page = page;
 			bvec->bv_offset = offset;
 
-			bvec->bv_len = len < (PAGE_SIZE - offset) ? len : (PAGE_SIZE - offset);
+			/*
+			 * One bio_vec per physically contiguous extent: a
+			 * whole 2M block when the umem is hugepage-backed,
+			 * a single page otherwise.
+			 */
+			contig = pcs_umem_contig(umem, pos);
+			bvec->bv_len = len < contig ? len : contig;
 
 			pos += bvec->bv_len;
 			len -= bvec->bv_len;
diff --git a/fs/fuse/kio/pcs/pcs_mr.c b/fs/fuse/kio/pcs/pcs_mr.c
index 1e3a74033510..58f1ff41c750 100644
--- a/fs/fuse/kio/pcs/pcs_mr.c
+++ b/fs/fuse/kio/pcs/pcs_mr.c
@@ -7,20 +7,30 @@
 #include <linux/slab.h>
 #include <linux/sched/mm.h>
 #include <linux/resource.h>
+#include <linux/mm.h>
+#include <linux/pgtable.h>
 
 #include "pcs_mr.h"
 
 void pcs_umem_release(struct pcs_umem *umem)
 {
 	struct mm_struct *mm_s = umem->mm;
-	int i, npages = umem->npages;
-
-	for (i = 0; npages; i++) {
-		int to_free = min_t(int, PCS_PAGES_PER_CHUNK, npages);
-
-		unpin_user_pages(umem->page_chunk[i].pages, to_free);
-		kfree(umem->page_chunk[i].pages);
-		npages -= to_free;
+	int npages = umem->npages;
+	int i;
+
+	for (i = 0; umem->page_chunk && i < umem->nchunks; i++) {
+		struct pcs_page_chunk *chunk = &umem->page_chunk[i];
+		int n = min_t(int, PCS_PAGES_PER_CHUNK, npages);
+
+		if (umem->page_shift == PMD_SHIFT) {
+			/* one full hugepage folio per chunk */
+			unpin_user_page_range_dirty_lock(chunk->head,
+					PCS_PAGES_PER_CHUNK, false);
+		} else if (chunk->pages) {
+			unpin_user_pages(chunk->pages, n);
+			kfree(chunk->pages);
+		}
+		npages -= n;
 	}
 	atomic64_sub(umem->npages, &mm_s->pinned_vm);
 	mmdrop(mm_s);
@@ -28,6 +38,43 @@ void pcs_umem_release(struct pcs_umem *umem)
 	kfree(umem);
 }
 
+/*
+ * A page chunk covers exactly one hugepage. If the whole pinned range
+ * is 2M-aligned and every chunk is backed by one aligned hugepage folio
+ * (hugetlbfs or fully-mapped THP), drop the 4k page arrays and keep just
+ * the folio head page in each chunk, raising page_shift accordingly.
+ * Mixed or misaligned mappings simply stay at 4k granularity.
+ */
+static void pcs_umem_try_compact(struct pcs_umem *umem)
+{
+	int i, j;
+
+	if (umem->fp_addr & (PMD_SIZE - 1))
+		return;
+	if (!umem->npages || umem->npages % PCS_PAGES_PER_CHUNK)
+		return;
+
+	for (i = 0; i < umem->nchunks; i++) {
+		struct page *head = umem->page_chunk[i].pages[0];
+		struct folio *folio = page_folio(head);
+
+		if (folio_nr_pages(folio) - folio_page_idx(folio, head) <
+				PCS_PAGES_PER_CHUNK)
+			return;
+		for (j = 1; j < PCS_PAGES_PER_CHUNK; j++)
+			if (umem->page_chunk[i].pages[j] != nth_page(head, j))
+				return;
+	}
+
+	for (i = 0; i < umem->nchunks; i++) {
+		struct page *head = umem->page_chunk[i].pages[0];
+
+		kfree(umem->page_chunk[i].pages);
+		umem->page_chunk[i].head = head;
+	}
+	umem->page_shift = PMD_SHIFT;
+}
+
 /*
  * Pin pages from userspace memory
  */
@@ -36,11 +83,10 @@ struct pcs_umem *pcs_umem_get(u64 start, u64 len)
 	struct pcs_umem *umem = NULL;
 	struct mm_struct *mm_s;
 	u64 fp_va;
-	int npages, nchunks, i, ret;
+	int npages, i, ret;
 
 	fp_va = start & PAGE_MASK;
 	npages = PAGE_ALIGN(start + len - fp_va) >> PAGE_SHIFT;
-	nchunks = DIV_ROUND_UP(npages, PCS_PAGES_PER_CHUNK);
 
 	umem = kzalloc(sizeof(*umem), GFP_KERNEL);
 	if (!umem)
@@ -49,12 +95,14 @@ struct pcs_umem *pcs_umem_get(u64 start, u64 len)
 	/*  pin user pages */
 	mm_s = current->mm;
 	umem->mm = mm_s;
+	umem->page_shift = PAGE_SHIFT;
+	umem->nchunks = DIV_ROUND_UP(npages, PCS_PAGES_PER_CHUNK);
 
 	mmgrab(mm_s);
-	mmap_read_lock(mm_s);
 
 	umem->fp_addr = fp_va;
-	umem->page_chunk = kcalloc(nchunks, sizeof(struct pcs_page_chunk), GFP_KERNEL);
+	umem->page_chunk = kcalloc(umem->nchunks, sizeof(struct pcs_page_chunk),
+				   GFP_KERNEL);
 	if (!umem->page_chunk) {
 		ret = -ENOMEM;
 		goto out_err;
@@ -71,9 +119,13 @@ struct pcs_umem *pcs_umem_get(u64 start, u64 len)
 		umem->page_chunk[i].pages = pages;
 
 		while (n) {
-			ret = pin_user_pages(fp_va, n, FOLL_WRITE | FOLL_LONGTERM, pages);
-			if (ret < 0)
+			ret = pin_user_pages_fast(fp_va, n,
+						  FOLL_WRITE | FOLL_LONGTERM, pages);
+			if (ret <= 0) {
+				if (!ret)
+					ret = -EFAULT;
 				goto out_err;
+			}
 
 			atomic64_add(ret, &mm_s->pinned_vm);
 			umem->npages += ret;
@@ -83,12 +135,12 @@ struct pcs_umem *pcs_umem_get(u64 start, u64 len)
 			npages -= ret;
 		}
 	}
-	mmap_read_unlock(mm_s);
+
+	pcs_umem_try_compact(umem);
 
 	return umem;
 
 out_err:
-	mmap_read_unlock(mm_s);
 	pcs_umem_release(umem);
 
 	return ERR_PTR(ret);
diff --git a/fs/fuse/kio/pcs/pcs_mr.h b/fs/fuse/kio/pcs/pcs_mr.h
index 64f237f57dec..ea02af1dd622 100644
--- a/fs/fuse/kio/pcs/pcs_mr.h
+++ b/fs/fuse/kio/pcs/pcs_mr.h
@@ -6,12 +6,14 @@
 #define _PCS_MR_H_ 1
 
 #include <linux/types.h>
+#include <linux/mm.h>
 #include <linux/highmem-internal.h>
 
 struct pcs_umem;
 
 #define PCS_MAX_MR		0x10000
-#define PCS_PAGE_CHUNK_SHIFT 9
+/* A page chunk covers exactly one hugepage: 2M, PMD_SIZE */
+#define PCS_PAGE_CHUNK_SHIFT (PMD_SHIFT - PAGE_SHIFT)
 #define PCS_PAGES_PER_CHUNK (1 << PCS_PAGE_CHUNK_SHIFT)
 
 struct pcs_mr_set {
@@ -20,14 +22,30 @@ struct pcs_mr_set {
 	atomic_t	mr_num;		/* number of registered MRs*/
 };
 
+/*
+ * A chunk covers exactly one 2M hugepage of VA: PCS_PAGES_PER_CHUNK 4k
+ * pages. Depending on umem->page_shift it stores either an array of
+ * the pinned 4k pages, or — when the chunk is backed by one aligned
+ * hugepage folio — just the folio head page, with no array at all.
+ */
 struct pcs_page_chunk
 {
-	struct page **pages; /* array of pinned pages */
+	union {
+		struct page **pages; /* page_shift == PAGE_SHIFT */
+		struct page *head;   /* page_shift == PMD_SHIFT */
+	};
 };
 
 struct pcs_umem {
 	struct pcs_page_chunk *page_chunk;
-	int npages; /* number of pinned pages */
+	int nchunks; /* number of allocated page chunks */
+	int npages; /* number of pinned PAGE_SIZE pages */
+	/*
+	 * Granularity of the stored page pointers: PAGE_SHIFT (chunks
+	 * hold 4k page arrays) or PMD_SHIFT (chunks hold hugepage folio
+	 * heads).
+	 */
+	unsigned int page_shift;
 	u64 fp_addr; /* First page base address */
 	struct mm_struct *mm;	/* mm the memory belongs to */
 };
@@ -47,13 +65,28 @@ struct pcs_mr {
  */
 static inline struct page *pcs_umem_page(struct pcs_umem *umem, u64 addr)
 {
-	unsigned int page_idx = (addr - umem->fp_addr) >> PAGE_SHIFT;
+	u64 off = addr - umem->fp_addr;
+	unsigned int page_idx = off >> PAGE_SHIFT;
 	unsigned int chunk_idx = page_idx >> PCS_PAGE_CHUNK_SHIFT;
 	unsigned int page_in_chunk = page_idx & (PCS_PAGES_PER_CHUNK - 1);
 
-	if (likely(page_idx < umem->npages))
+	if (unlikely(page_idx >= umem->npages))
+		return NULL;
+
+	if (umem->page_shift == PAGE_SHIFT)
 		return umem->page_chunk[chunk_idx].pages[page_in_chunk];
-	return NULL;
+	return nth_page(umem->page_chunk[chunk_idx].head, page_in_chunk);
+}
+
+/*
+ * Number of physically contiguous bytes mapped at the address: from addr
+ * to the end of its backing unit (PAGE_SIZE or a 2M hugepage folio).
+ * Lets callers build bio_vecs spanning a whole hugepage instead of
+ * one bio_vec per 4k page.
+ */
+static inline unsigned int pcs_umem_contig(struct pcs_umem *umem, u64 addr)
+{
+	return (1U << umem->page_shift) - (addr & ((1U << umem->page_shift) - 1));
 }
 
 struct pcs_umem *pcs_umem_get(u64 start, u64 len);

-- 
Git-155)

_______________________________________________
Devel mailing list
Devel@openvz.org
https://lists.openvz.org/mailman/listinfo/devel