OpenVZ / Virtuozzo kernel development (devel@openvz.org)
 help / color / mirror / Atom feed
From: Andrey Zhadchenko <andrey.zhadchenko@virtuozzo.com>
To: khorenko@virtuozzo.com, alexander.ivanov@virtuozzo.com
Cc: devel@openvz.org
Subject: [Devel] [PATCH hci-8.1 v2 6/7] dm-qcow2: process fast path qios in per-cpu batches on a workqueue
Date: Tue, 10 Nov 2026 14:46:28 +0300	[thread overview]
Message-ID: <20261110114629.416695-7-andrey.zhadchenko@virtuozzo.com> (raw)
In-Reply-To: <20261110114629.416695-1-andrey.zhadchenko@virtuozzo.com>

queue_work() is pretty costly under high load. It takes pool->lock;
unbound pools do not track running workers (WORKER_UNBOUND is part
of WORKER_NOT_RUNNING), so an idle kworker is woken (idle exit + IPI +
context switch) even if a running worker would pick the work up right
away. Every qio has its own work, so this is paid for every I/O. With
several submitters this ends up in several context switches per I/O
and in contention on pool->lock.

Batch requests with percpu llists. Additionally track if the worker
is busy and queue_work() only if it is not. The worker processes one
batch per run and requeues itself if more qios came meanwhile, so
flush_workqueue(), e.g. on backward merge completion with I/O still
running, does not wait for a worker which is refilled all the time.

The fast path is now serial per submitting CPU: qios queued from one
CPU are processed one by one by its work, while qios from different
CPUs are still processed in parallel. This is fine, since a fast path
qio is only mapped and submitted as direct AIO: O_DIRECT is enforced
for every delta. The exception is a partial discard, which punches
the hole synchronously in the worker, so the qios queued after it on
that CPU wait. Most of them would wait anyway: on ext4 and xfs punch
hole takes the inode lock exclusively, and direct I/O submission to
the same image sleeps on it.

https://virtuozzo.atlassian.net/browse/VSTOR-95494
Feature: dm-qcow2: block device over QCOW2 files driver
Signed-off-by: Andrey Zhadchenko <andrey.zhadchenko@virtuozzo.com>
---
v2: moved before "declare DM_TARGET_NOWAIT"; process one batch per run
    and requeue, so flush_workqueue() does not wait for a work which is
    refilled all the time; commit message: fixed the claim about
    re-queued qios, describes that the fast path is serial per
    submitting CPU and that partial discards punch the hole in the
    worker, measurements moved to the thread

 drivers/md/dm-qcow2-map.c    | 100 +++++++++++++++++++++++++++++++----
 drivers/md/dm-qcow2-target.c |  10 +++-
 drivers/md/dm-qcow2.h        |   6 ++-
 3 files changed, 103 insertions(+), 13 deletions(-)

diff --git a/drivers/md/dm-qcow2-map.c b/drivers/md/dm-qcow2-map.c
index 7a8deba4b0a83..a87a396af56d0 100644
--- a/drivers/md/dm-qcow2-map.c
+++ b/drivers/md/dm-qcow2-map.c
@@ -87,6 +87,7 @@ static void process_read_qio(struct qcow2 *qcow2, struct qio *qio,
 			     struct qcow2_map *map);
 static void init_qrq_and_embedded_qio(struct qcow2_target *tgt, struct request *rq,
 				      struct qcow2_rq *qrq, struct qio *qio);
+static void qcow2_fast_submit(struct qcow2_target *tgt, struct qio *qio);
 
 static loff_t bytes_off_in_cluster(struct qcow2 *qcow2, struct qio *qio)
 {
@@ -362,8 +363,6 @@ struct qio *qcow2_alloc_qio(mempool_t *pool, bool zero)
 }
 ALLOW_ERROR_INJECTION(qcow2_alloc_qio, NULL);
 
-static void do_qio_work(struct work_struct *ws);
-
 void qcow2_init_qio(struct qio *qio, unsigned int bi_op, struct qcow2 *qcow2)
 {
 	qio->qcow2 = qcow2;
@@ -376,7 +375,6 @@ void qcow2_init_qio(struct qio *qio, unsigned int bi_op, struct qcow2 *qcow2)
 	qio->ref_index = REF_INDEX_INVALID;
 	qio->state = QIO_STATE_START;
 	atomic_set(&qio->remaining, 1);
-	INIT_WORK(&qio->worker, do_qio_work);
 
 	/*
 	 * Initially set into BLK_STS_OK, while aio complete,
@@ -4511,7 +4509,7 @@ static void qcow2_submit_embedded_qio(struct qcow2_target *tgt, struct qio *qio)
 
 	if (blk_rq_bytes(rq)) {
 		queue_list_id = QLIST_EMBEDDED;
-		worker = &qio->worker;
+		worker = NULL;
 	} else {
 		WARN_ON_ONCE(qio->bi_op != REQ_OP_FLUSH);
 		queue_list_id = QLIST_FLUSH;
@@ -4534,8 +4532,13 @@ static void qcow2_submit_embedded_qio(struct qcow2_target *tgt, struct qio *qio)
 	}
 	spin_unlock_irqrestore(&qcow2->deferred_lock, flags);
 
-	if (queue)
+	if (!queue)
+		return;
+
+	if (worker)
 		queue_work(tgt->wq, worker);
+	else
+		qcow2_fast_submit(tgt, qio);
 }
 
 void qcow2_submit_embedded_qios(struct qcow2_target *tgt, struct list_head *list)
@@ -4634,15 +4637,11 @@ static void qio_try_fast(struct qio *qio, struct list_head *deferred_qios)
 		list_add_tail(&qio->link, deferred_qios);
 }
 
-static void do_qio_work(struct work_struct *ws)
+static void do_qio_work(struct qio *qio)
 {
-	struct qio *qio = container_of(ws, struct qio, worker);
-	struct qio *i, *next;
 	struct qcow2 *qcow2 = qio->qcow2;
 	struct qcow2_target *tgt = qcow2->tgt;
-	unsigned int pflags = current->flags;
-
-	current->flags |= PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO;
+	struct qio *i, *next;
 
 	switch (qio->state) {
 	case QIO_STATE_START: {
@@ -4668,8 +4667,87 @@ static void do_qio_work(struct work_struct *ws)
 		qio_endio(qio);
 		break;
 	}
+}
+
+struct qcow2_fast_cpu {
+	struct llist_head qios;
+	struct work_struct work;
+	struct qcow2_target *tgt;
+	atomic_t busy;
+};
+
+static void qcow2_fast_submit(struct qcow2_target *tgt, struct qio *qio)
+{
+	struct qcow2_fast_cpu *fc = get_cpu_ptr(tgt->fast);
+
+	llist_add(&qio->fast_link, &fc->qios);
+	/* Pairs with smp_mb() in qcow2_fast_work() */
+	smp_mb();
+	if (!atomic_read(&fc->busy) && atomic_cmpxchg(&fc->busy, 0, 1) == 0)
+		queue_work(tgt->wq, &fc->work);
+	put_cpu_ptr(tgt->fast);
+}
+
+static void qcow2_fast_work(struct work_struct *work)
+{
+	struct qcow2_fast_cpu *fc = container_of(work, struct qcow2_fast_cpu, work);
+	unsigned int pflags = current->flags;
+	struct llist_node *node;
+	struct qio *qio, *next;
+
+	current->flags |= PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO;
+
+	node = llist_del_all(&fc->qios);
+	node = llist_reverse_order(node);
+	llist_for_each_entry_safe(qio, next, node, fast_link)
+		do_qio_work(qio);
 
 	current_restore_flags(pflags, PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO);
+
+	if (llist_empty(&fc->qios)) {
+		atomic_set(&fc->busy, 0);
+		/* just to be sure we look at the list after busy is posted */
+		smp_mb();
+		if (llist_empty(&fc->qios))
+			return;
+		if (atomic_cmpxchg(&fc->busy, 0, 1) != 0)
+			return;
+	}
+	queue_work(fc->tgt->wq, &fc->work);
+}
+
+void qcow2_fast_exit(struct qcow2_target *tgt)
+{
+	struct qcow2_fast_cpu *fc;
+	int cpu;
+
+	if (!tgt->fast)
+		return;
+	for_each_possible_cpu(cpu) {
+		fc = per_cpu_ptr(tgt->fast, cpu);
+		WARN_ON_ONCE(!llist_empty(&fc->qios));
+	}
+	free_percpu(tgt->fast);
+	tgt->fast = NULL;
+}
+
+int qcow2_fast_init(struct qcow2_target *tgt)
+{
+	struct qcow2_fast_cpu *fc;
+	int cpu;
+
+	tgt->fast = alloc_percpu(struct qcow2_fast_cpu);
+	if (!tgt->fast)
+		return -ENOMEM;
+	for_each_possible_cpu(cpu) {
+		fc = per_cpu_ptr(tgt->fast, cpu);
+		init_llist_head(&fc->qios);
+		INIT_WORK(&fc->work, qcow2_fast_work);
+		fc->tgt = tgt;
+		atomic_set(&fc->busy, 0);
+	}
+
+	return 0;
 }
 
 struct qio_data_llseek_hole {
diff --git a/drivers/md/dm-qcow2-target.c b/drivers/md/dm-qcow2-target.c
index 8337efc5c6238..3e85de5c8b050 100644
--- a/drivers/md/dm-qcow2-target.c
+++ b/drivers/md/dm-qcow2-target.c
@@ -253,6 +253,7 @@ static void qcow2_tgt_destroy(struct qcow2_target *tgt)
 		/* Now kill the queue */
 		destroy_workqueue(tgt->wq);
 	}
+	qcow2_fast_exit(tgt);
 	qcow2_merge_backward_cancel(tgt);
 
 	mempool_destroy(tgt->qio_pool);
@@ -477,6 +478,11 @@ static struct qcow2_target *alloc_qcow2_target(struct dm_target *ti)
 		goto out_pool;
 	}
 
+	if (qcow2_fast_init(tgt)) {
+		ti->error = "Can't alloc fast path slots";
+		goto out_wq;
+	}
+
 	for (i = 0; i < 2; i++) {
 		release = i ? inflight_ref_exit1 : inflight_ref_exit0;
 		if (percpu_ref_init(&tgt->inflight_ref[i], release,
@@ -484,7 +490,7 @@ static struct qcow2_target *alloc_qcow2_target(struct dm_target *ti)
 			if (i)
 				percpu_ref_exit(&tgt->inflight_ref[0]);
 			ti->error = "could not alloc percpu_ref";
-			goto out_wq;
+			goto out_fast;
 		}
 	}
 
@@ -504,6 +510,8 @@ static struct qcow2_target *alloc_qcow2_target(struct dm_target *ti)
 	qcow2_set_service_operations(ti, false);
 
 	return tgt;
+out_fast:
+	qcow2_fast_exit(tgt);
 out_wq:
 	destroy_workqueue(tgt->wq);
 out_pool:
diff --git a/drivers/md/dm-qcow2.h b/drivers/md/dm-qcow2.h
index c044c630ca238..d6884eba2f9de 100644
--- a/drivers/md/dm-qcow2.h
+++ b/drivers/md/dm-qcow2.h
@@ -4,6 +4,7 @@
 
 #include <linux/percpu-refcount.h>
 #include <linux/device-mapper.h>
+#include <linux/llist.h>
 #include <linux/fs.h>
 #include <linux/eventfd.h>
 
@@ -192,6 +193,7 @@ struct qcow2_target {
 	 */
 	struct qcow2 *top;
 	struct workqueue_struct *wq;
+	struct qcow2_fast_cpu __percpu *fast;
 
 	struct completion inflight_ref_comp;
 	struct percpu_ref inflight_ref[2];
@@ -377,7 +379,7 @@ struct qio {
 	struct qcow2 *qcow2;
 	qcow2_endio_t endio_cb;
 	void *endio_cb_data;
-	struct work_struct worker;
+	struct llist_node fast_link;
 	int state;
 };
 
@@ -394,6 +396,8 @@ loff_t qcow2_llseek_hole(struct dm_target *ti, loff_t offset, int whence);
 
 void do_qcow2_work(struct work_struct *ws);
 void do_qcow2_fsync_work(struct work_struct *ws);
+int qcow2_fast_init(struct qcow2_target *tgt);
+void qcow2_fast_exit(struct qcow2_target *tgt);
 int qcow2_alloc_and_insert_md_page(struct qcow2 *qcow2, u64 index,
 					struct md_page **md);
 struct md_page *qcow2_md_page_find_or_postpone(struct qcow2 *qcow2,
-- 
2.43.5

_______________________________________________
Devel mailing list
Devel@openvz.org
https://lists.openvz.org/mailman/listinfo/devel

  parent reply	other threads:[~2026-11-10 11:52 UTC|newest]

Thread overview: 26+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-10 11:46 [Devel] [PATCH hci-8.1 v2 0/7] dm-qcow2: per-qio fast path Andrey Zhadchenko
2026-11-10 11:46 ` [Devel] [PATCH hci-8.1 v2 1/7] dm-qcow2: repeat md page lookup if it was inserted in parallel Andrey Zhadchenko
2026-11-10 11:55   ` no-reply
2026-11-10 17:56   ` Konstantin Khorenko
2026-11-10 11:46 ` [Devel] [PATCH hci-8.1 v2 2/7] dm-qcow2: wait for L2 writeback in perform_rw_mapped() Andrey Zhadchenko
2026-11-10 11:56   ` no-reply
2026-11-10 17:56   ` Konstantin Khorenko
2026-11-10 11:46 ` [Devel] [PATCH hci-8.1 v2 3/7] dm-qcow2: prepare process_one_qio() for fastpath Andrey Zhadchenko
2026-11-10 11:59   ` no-reply
2026-11-10 13:50     ` Andrey Zhadchenko
2026-11-10 17:56   ` Konstantin Khorenko
2026-11-11 14:36   ` Alexander Ivanov
2026-11-10 11:46 ` [Devel] [PATCH hci-8.1 v2 4/7] dm-qcow2: add per-qio work infrastructure Andrey Zhadchenko
2026-11-10 12:02   ` no-reply
2026-11-11 15:34     ` Andrey Zhadchenko
2026-11-10 17:56   ` Konstantin Khorenko
2026-11-10 11:46 ` [Devel] [PATCH hci-8.1 v2 5/7] dm-qcow2: implement per-qio fast processing Andrey Zhadchenko
2026-11-10 12:08   ` no-reply
2026-11-10 17:56   ` Konstantin Khorenko
2026-11-10 11:46 ` Andrey Zhadchenko [this message]
2026-11-10 12:06   ` [Devel] [PATCH hci-8.1 v2 6/7] dm-qcow2: process fast path qios in per-cpu batches on a workqueue no-reply
2026-11-10 17:56   ` Konstantin Khorenko
2026-11-11 15:00   ` Alexander Ivanov
2026-11-10 11:46 ` [Devel] [PATCH hci-8.1 v2 7/7] dm-qcow2: declare DM_TARGET_NOWAIT Andrey Zhadchenko
2026-11-10 12:06   ` no-reply
2026-11-10 17:56 ` [Devel] [PATCH hci-8.1 v2 0/7] dm-qcow2: per-qio fast path Konstantin Khorenko

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261110114629.416695-7-andrey.zhadchenko@virtuozzo.com \
    --to=andrey.zhadchenko@virtuozzo.com \
    --cc=alexander.ivanov@virtuozzo.com \
    --cc=devel@openvz.org \
    --cc=khorenko@virtuozzo.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox