[Devel] [PATCH hci-8.1 v7 7/8] block/blk-cbt: don't sleep in CBT for a REQ_NOWAIT write

[Pavel Tikhomirov <ptikhomirov@virtuozzo.com>]

Accounting a write in CBT may allocate a bitmap page with GFP_NOIO,
either for the write itself or for the extent it evicts from the per-cpu
cache, so a REQ_NOWAIT bio can sleep in submit_bio_noacct().

For a REQ_NOWAIT write, skip the cache and set the bits directly, which
doesn't allocate when all the pages are there. If one is missing, fail
the bio with BLK_STS_AGAIN, for the submitter to retry it in a context
that may block. The throttle dispatch work may sleep, so it doesn't use
this.

https://virtuozzo.atlassian.net/browse/VSTOR-149268
Feature: cbt: changed block tracking (for backup)
Signed-off-by: Pavel Tikhomirov <ptikhomirov@virtuozzo.com>
---
 block/blk-cbt.c        | 57 +++++++++++++++++++++++++++++++++++++++++++-------
 block/blk-core.c       |  5 ++++-
 block/blk-throttle.c   |  2 +-
 include/linux/blkdev.h |  7 +++++--
 4 files changed, 59 insertions(+), 12 deletions(-)

diff --git a/block/blk-cbt.c b/block/blk-cbt.c
index 3ad4b9a25e1f4..7b6441fad08f9 100644
--- a/block/blk-cbt.c
+++ b/block/blk-cbt.c
@@ -231,26 +231,55 @@ static int __blk_cbt_set(struct cbt_info  *cbt, blkcnt_t block,
 	return (pages_missed && *pages_missed) ? -EAGAIN : 0;
 }
 
-static void blk_cbt_add(struct cbt_info *cbt, blkcnt_t start, blkcnt_t len)
+/*
+ * Return true if all the map pages for [block, block + count) are there,
+ * or for an empty or out of range request, which needs no page.
+ */
+static bool cbt_pages_present(struct cbt_info *cbt, blkcnt_t block,
+			      blkcnt_t count)
+{
+	unsigned long idx, last;
+
+	if (unlikely(!count || block + count > cbt->block_max))
+		return true;
+
+	last = (block + count - 1) / BITS_PER_PAGE;
+	for (idx = block / BITS_PER_PAGE; idx <= last; idx++)
+		if (!CBT_PAGE(cbt, idx))
+			return false;
+
+	return true;
+}
+
+/* Return true if nowait and a bitmap page is missing */
+static bool blk_cbt_add(struct cbt_info *cbt, blkcnt_t start, blkcnt_t len,
+			bool nowait)
 {
 	struct cbt_extent *ex;
 	struct cbt_extent old;
 	blkcnt_t end;
 
 	if (unlikely(test_bit(CBT_ERROR, &cbt->flags)))
-		return;
+		return false;
 	end = DIV_ROUND_UP(start + len, 1UL << cbt->block_bits);
 	start >>= cbt->block_bits;
 	len = end - start;
+	if (unlikely(nowait)) {
+		/* Don't sleep: set the bits directly, only with all pages there */
+		if (!cbt_pages_present(cbt, start, len))
+			return true;
+		__blk_cbt_set(cbt, start, len, 1, NULL, NULL);
+		return false;
+	}
 	if (unlikely(test_bit(CBT_NOCACHE, &cbt->flags))) {
 		__blk_cbt_set(cbt, start, len, 1, NULL, NULL);
-		return;
+		return false;
 	}
 	ex = get_cpu_ptr(cbt->cache);
 	if (ex->start + ex->len == start) {
 		ex->len += len;
 		put_cpu_ptr(cbt->cache);
-		return;
+		return false;
 	}
 	old = *ex;
 	ex->start = start;
@@ -259,21 +288,33 @@ static void blk_cbt_add(struct cbt_info *cbt, blkcnt_t start, blkcnt_t len)
 
 	if (likely(old.len))
 		__blk_cbt_set(cbt, old.start, old.len, 1, NULL, NULL);
+	return false;
 }
 
-void blk_cbt_bio_queue(struct request_queue *q, struct bio *bio)
+/*
+ * Account a write in CBT. With nowait, don't sleep: return true if a bitmap
+ * page is missing, for the caller to fail the bio with BLK_STS_AGAIN.
+ */
+bool blk_cbt_bio_queue(struct request_queue *q, struct bio *bio, bool nowait)
 {
 	struct cbt_info *cbt;
+	bool wouldblock = false;
 	int idx;
 
 	if (list_empty(&q->cbt_list) || bio_data_dir(bio) == READ || !bio->bi_iter.bi_size)
-		return;
+		return false;
 
 	idx = srcu_read_lock(&cbt_srcu);
 	list_for_each_entry_srcu(cbt, &q->cbt_list, list,
-				 srcu_read_lock_held(&cbt_srcu))
-		blk_cbt_add(cbt, bio->bi_iter.bi_sector << 9, bio->bi_iter.bi_size);
+				 srcu_read_lock_held(&cbt_srcu)) {
+		if (blk_cbt_add(cbt, bio->bi_iter.bi_sector << 9,
+				bio->bi_iter.bi_size, nowait)) {
+			wouldblock = true;
+			break;
+		}
+	}
 	srcu_read_unlock(&cbt_srcu, idx);
+	return wouldblock;
 }
 
 static struct cbt_info *do_cbt_alloc(struct request_queue *q, __u8 *name,
diff --git a/block/blk-core.c b/block/blk-core.c
index bcc3060de670c..6bc78f7c23d38 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -871,7 +871,10 @@ void submit_bio_noacct(struct bio *bio)
 
 	if (blk_throtl_bio(bio))
 		return;
-	blk_cbt_bio_queue(q, bio);
+	if (blk_cbt_bio_queue(q, bio, bio->bi_opf & REQ_NOWAIT)) {
+		bio_wouldblock_error(bio);
+		return;
+	}
 	submit_bio_noacct_nocheck(bio);
 	return;
 
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index e1e008a30bf2c..00f0db0a57f8d 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -1225,7 +1225,7 @@ static void blk_throtl_dispatch_work_fn(struct work_struct *work)
 		blk_start_plug(&plug);
 		while ((bio = bio_list_pop(&bio_list_on_stack))) {
 			/* submit_bio_noacct() did not account it in CBT */
-			blk_cbt_bio_queue(q, bio);
+			blk_cbt_bio_queue(q, bio, false);
 			submit_bio_noacct_nocheck(bio);
 		}
 		blk_finish_plug(&plug);
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 1284bb0e4ebc0..885555f480462 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -1626,7 +1626,8 @@ static inline bool blk_crypto_register(struct blk_crypto_profile *profile,
 #if defined (CONFIG_BLK_DEV_CBT)
 extern void blk_cbt_update_size(struct block_device *bdev);
 extern void blk_cbt_release(struct request_queue *q);
-extern void blk_cbt_bio_queue(struct request_queue *q, struct bio *bio);
+extern bool blk_cbt_bio_queue(struct request_queue *q, struct bio *bio,
+			      bool nowait);
 extern int blk_cbt_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg);
 #else /* CONFIG_BLK_DEV_CBT */
 static inline void blk_cbt_update_size(struct block_device *bdev)
@@ -1635,8 +1636,10 @@ static inline void blk_cbt_update_size(struct block_device *bdev)
 static inline void blk_cbt_release(struct request_queue *q)
 {
 }
-static inline void blk_cbt_bio_queue(struct request_queue *q, struct bio *bio)
+static inline bool blk_cbt_bio_queue(struct request_queue *q, struct bio *bio,
+				     bool nowait)
 {
+	return false;
 }
 static inline int blk_cbt_ioctl(struct block_device *bdev, unsigned cmd,
 				char __user *arg)

-- 
2.55.0

_______________________________________________
Devel mailing list
Devel@openvz.org
https://lists.openvz.org/mailman/listinfo/devel