All Virtuozzo development lists (kernel + QEMU)
 help / color / mirror / Atom feed
From: Alexander Ivanov <alexander.ivanov@virtuozzo.com>
To: Andrey Zhadchenko <andrey.zhadchenko@virtuozzo.com>,
	khorenko@virtuozzo.com
Cc: devel@openvz.org
Subject: Re: [Devel] [PATCH hci-8.1 v2 6/7] dm-qcow2: process fast path qios in per-cpu batches on a workqueue
Date: Wed, 11 Nov 2026 16:00:13 +0100	[thread overview]
Message-ID: <62f1f094-a00e-4573-ba0b-2b323e73b2b9@virtuozzo.com> (raw)
In-Reply-To: <20261110114629.416695-7-andrey.zhadchenko@virtuozzo.com>


On 11/10/26 12:46, Andrey Zhadchenko wrote:
> queue_work() is pretty costly under high load. It takes pool->lock;
> unbound pools do not track running workers (WORKER_UNBOUND is part
> of WORKER_NOT_RUNNING), so an idle kworker is woken (idle exit + IPI +
> context switch) even if a running worker would pick the work up right
> away. Every qio has its own work, so this is paid for every I/O. With
> several submitters this ends up in several context switches per I/O
> and in contention on pool->lock.
>
> Batch requests with percpu llists. Additionally track if the worker
> is busy and queue_work() only if it is not. The worker processes one
> batch per run and requeues itself if more qios came meanwhile, so
> flush_workqueue(), e.g. on backward merge completion with I/O still
> running, does not wait for a worker which is refilled all the time.
>
> The fast path is now serial per submitting CPU: qios queued from one
> CPU are processed one by one by its work, while qios from different
> CPUs are still processed in parallel. This is fine, since a fast path
> qio is only mapped and submitted as direct AIO: O_DIRECT is enforced
> for every delta. The exception is a partial discard, which punches
> the hole synchronously in the worker, so the qios queued after it on
> that CPU wait. Most of them would wait anyway: on ext4 and xfs punch
> hole takes the inode lock exclusively, and direct I/O submission to
> the same image sleeps on it.
>
> https://virtuozzo.atlassian.net/browse/VSTOR-95494
> Feature: dm-qcow2: block device over QCOW2 files driver
> Signed-off-by: Andrey Zhadchenko <andrey.zhadchenko@virtuozzo.com>
> ---
> v2: moved before "declare DM_TARGET_NOWAIT"; process one batch per run
>      and requeue, so flush_workqueue() does not wait for a work which is
>      refilled all the time; commit message: fixed the claim about
>      re-queued qios, describes that the fast path is serial per
>      submitting CPU and that partial discards punch the hole in the
>      worker, measurements moved to the thread
>
>   drivers/md/dm-qcow2-map.c    | 100 +++++++++++++++++++++++++++++++----
>   drivers/md/dm-qcow2-target.c |  10 +++-
>   drivers/md/dm-qcow2.h        |   6 ++-
>   3 files changed, 103 insertions(+), 13 deletions(-)
>
> diff --git a/drivers/md/dm-qcow2-map.c b/drivers/md/dm-qcow2-map.c
> index 7a8deba4b0a83..a87a396af56d0 100644
> --- a/drivers/md/dm-qcow2-map.c
> +++ b/drivers/md/dm-qcow2-map.c
> @@ -87,6 +87,7 @@ static void process_read_qio(struct qcow2 *qcow2, struct qio *qio,
>   			     struct qcow2_map *map);
>   static void init_qrq_and_embedded_qio(struct qcow2_target *tgt, struct request *rq,
>   				      struct qcow2_rq *qrq, struct qio *qio);
> +static void qcow2_fast_submit(struct qcow2_target *tgt, struct qio *qio);
>   
>   static loff_t bytes_off_in_cluster(struct qcow2 *qcow2, struct qio *qio)
>   {
> @@ -362,8 +363,6 @@ struct qio *qcow2_alloc_qio(mempool_t *pool, bool zero)
>   }
>   ALLOW_ERROR_INJECTION(qcow2_alloc_qio, NULL);
>   
> -static void do_qio_work(struct work_struct *ws);
> -
>   void qcow2_init_qio(struct qio *qio, unsigned int bi_op, struct qcow2 *qcow2)
>   {
>   	qio->qcow2 = qcow2;
> @@ -376,7 +375,6 @@ void qcow2_init_qio(struct qio *qio, unsigned int bi_op, struct qcow2 *qcow2)
>   	qio->ref_index = REF_INDEX_INVALID;
>   	qio->state = QIO_STATE_START;
>   	atomic_set(&qio->remaining, 1);
> -	INIT_WORK(&qio->worker, do_qio_work);
>   
>   	/*
>   	 * Initially set into BLK_STS_OK, while aio complete,
> @@ -4511,7 +4509,7 @@ static void qcow2_submit_embedded_qio(struct qcow2_target *tgt, struct qio *qio)
>   
>   	if (blk_rq_bytes(rq)) {
>   		queue_list_id = QLIST_EMBEDDED;
> -		worker = &qio->worker;
> +		worker = NULL;
>   	} else {
>   		WARN_ON_ONCE(qio->bi_op != REQ_OP_FLUSH);
>   		queue_list_id = QLIST_FLUSH;
> @@ -4534,8 +4532,13 @@ static void qcow2_submit_embedded_qio(struct qcow2_target *tgt, struct qio *qio)
>   	}
>   	spin_unlock_irqrestore(&qcow2->deferred_lock, flags);
>   
> -	if (queue)
> +	if (!queue)
> +		return;
> +
> +	if (worker)
>   		queue_work(tgt->wq, worker);
> +	else
> +		qcow2_fast_submit(tgt, qio);
>   }
>   
>   void qcow2_submit_embedded_qios(struct qcow2_target *tgt, struct list_head *list)
> @@ -4634,15 +4637,11 @@ static void qio_try_fast(struct qio *qio, struct list_head *deferred_qios)
>   		list_add_tail(&qio->link, deferred_qios);
>   }
>   
> -static void do_qio_work(struct work_struct *ws)
> +static void do_qio_work(struct qio *qio)
>   {
> -	struct qio *qio = container_of(ws, struct qio, worker);
> -	struct qio *i, *next;
>   	struct qcow2 *qcow2 = qio->qcow2;
>   	struct qcow2_target *tgt = qcow2->tgt;
> -	unsigned int pflags = current->flags;
> -
> -	current->flags |= PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO;
> +	struct qio *i, *next;
>   
>   	switch (qio->state) {
>   	case QIO_STATE_START: {
> @@ -4668,8 +4667,87 @@ static void do_qio_work(struct work_struct *ws)
>   		qio_endio(qio);
>   		break;
>   	}
> +}
> +
> +struct qcow2_fast_cpu {
> +	struct llist_head qios;
> +	struct work_struct work;
> +	struct qcow2_target *tgt;
> +	atomic_t busy;
> +};
> +
> +static void qcow2_fast_submit(struct qcow2_target *tgt, struct qio *qio)
> +{
> +	struct qcow2_fast_cpu *fc = get_cpu_ptr(tgt->fast);
> +
> +	llist_add(&qio->fast_link, &fc->qios);
> +	/* Pairs with smp_mb() in qcow2_fast_work() */
> +	smp_mb();
If I’m not mistaken, the try_cmpxchg() in llist_add() is already fully 
ordered, so the smp_mb() isn’t needed.
> +	if (!atomic_read(&fc->busy) && atomic_cmpxchg(&fc->busy, 0, 1) == 0)
> +		queue_work(tgt->wq, &fc->work);
> +	put_cpu_ptr(tgt->fast);
> +}
> +
> +static void qcow2_fast_work(struct work_struct *work)
> +{
> +	struct qcow2_fast_cpu *fc = container_of(work, struct qcow2_fast_cpu, work);
> +	unsigned int pflags = current->flags;
> +	struct llist_node *node;
> +	struct qio *qio, *next;
> +
> +	current->flags |= PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO;
> +
> +	node = llist_del_all(&fc->qios);
> +	node = llist_reverse_order(node);
> +	llist_for_each_entry_safe(qio, next, node, fast_link)
> +		do_qio_work(qio);
>   
>   	current_restore_flags(pflags, PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO);
> +
> +	if (llist_empty(&fc->qios)) {
> +		atomic_set(&fc->busy, 0);
> +		/* just to be sure we look at the list after busy is posted */
> +		smp_mb();
> +		if (llist_empty(&fc->qios))
> +			return;
> +		if (atomic_cmpxchg(&fc->busy, 0, 1) != 0)
> +			return;
> +	}
> +	queue_work(fc->tgt->wq, &fc->work);
> +}
> +
> +void qcow2_fast_exit(struct qcow2_target *tgt)
> +{
> +	struct qcow2_fast_cpu *fc;
> +	int cpu;
> +
> +	if (!tgt->fast)
> +		return;
> +	for_each_possible_cpu(cpu) {
> +		fc = per_cpu_ptr(tgt->fast, cpu);
> +		WARN_ON_ONCE(!llist_empty(&fc->qios));
> +	}
> +	free_percpu(tgt->fast);
> +	tgt->fast = NULL;
> +}
> +
> +int qcow2_fast_init(struct qcow2_target *tgt)
> +{
> +	struct qcow2_fast_cpu *fc;
> +	int cpu;
> +
> +	tgt->fast = alloc_percpu(struct qcow2_fast_cpu);
> +	if (!tgt->fast)
> +		return -ENOMEM;
> +	for_each_possible_cpu(cpu) {
> +		fc = per_cpu_ptr(tgt->fast, cpu);
> +		init_llist_head(&fc->qios);
> +		INIT_WORK(&fc->work, qcow2_fast_work);
> +		fc->tgt = tgt;
> +		atomic_set(&fc->busy, 0);
> +	}
> +
> +	return 0;
>   }
>   
>   struct qio_data_llseek_hole {
> diff --git a/drivers/md/dm-qcow2-target.c b/drivers/md/dm-qcow2-target.c
> index 8337efc5c6238..3e85de5c8b050 100644
> --- a/drivers/md/dm-qcow2-target.c
> +++ b/drivers/md/dm-qcow2-target.c
> @@ -253,6 +253,7 @@ static void qcow2_tgt_destroy(struct qcow2_target *tgt)
>   		/* Now kill the queue */
>   		destroy_workqueue(tgt->wq);
>   	}
> +	qcow2_fast_exit(tgt);
>   	qcow2_merge_backward_cancel(tgt);
>   
>   	mempool_destroy(tgt->qio_pool);
> @@ -477,6 +478,11 @@ static struct qcow2_target *alloc_qcow2_target(struct dm_target *ti)
>   		goto out_pool;
>   	}
>   
> +	if (qcow2_fast_init(tgt)) {
> +		ti->error = "Can't alloc fast path slots";
> +		goto out_wq;
> +	}
> +
>   	for (i = 0; i < 2; i++) {
>   		release = i ? inflight_ref_exit1 : inflight_ref_exit0;
>   		if (percpu_ref_init(&tgt->inflight_ref[i], release,
> @@ -484,7 +490,7 @@ static struct qcow2_target *alloc_qcow2_target(struct dm_target *ti)
>   			if (i)
>   				percpu_ref_exit(&tgt->inflight_ref[0]);
>   			ti->error = "could not alloc percpu_ref";
> -			goto out_wq;
> +			goto out_fast;
>   		}
>   	}
>   
> @@ -504,6 +510,8 @@ static struct qcow2_target *alloc_qcow2_target(struct dm_target *ti)
>   	qcow2_set_service_operations(ti, false);
>   
>   	return tgt;
> +out_fast:
> +	qcow2_fast_exit(tgt);
>   out_wq:
>   	destroy_workqueue(tgt->wq);
>   out_pool:
> diff --git a/drivers/md/dm-qcow2.h b/drivers/md/dm-qcow2.h
> index c044c630ca238..d6884eba2f9de 100644
> --- a/drivers/md/dm-qcow2.h
> +++ b/drivers/md/dm-qcow2.h
> @@ -4,6 +4,7 @@
>   
>   #include <linux/percpu-refcount.h>
>   #include <linux/device-mapper.h>
> +#include <linux/llist.h>
>   #include <linux/fs.h>
>   #include <linux/eventfd.h>
>   
> @@ -192,6 +193,7 @@ struct qcow2_target {
>   	 */
>   	struct qcow2 *top;
>   	struct workqueue_struct *wq;
> +	struct qcow2_fast_cpu __percpu *fast;
>   
>   	struct completion inflight_ref_comp;
>   	struct percpu_ref inflight_ref[2];
> @@ -377,7 +379,7 @@ struct qio {
>   	struct qcow2 *qcow2;
>   	qcow2_endio_t endio_cb;
>   	void *endio_cb_data;
> -	struct work_struct worker;
> +	struct llist_node fast_link;
>   	int state;
>   };
>   
> @@ -394,6 +396,8 @@ loff_t qcow2_llseek_hole(struct dm_target *ti, loff_t offset, int whence);
>   
>   void do_qcow2_work(struct work_struct *ws);
>   void do_qcow2_fsync_work(struct work_struct *ws);
> +int qcow2_fast_init(struct qcow2_target *tgt);
> +void qcow2_fast_exit(struct qcow2_target *tgt);
>   int qcow2_alloc_and_insert_md_page(struct qcow2 *qcow2, u64 index,
>   					struct md_page **md);
>   struct md_page *qcow2_md_page_find_or_postpone(struct qcow2 *qcow2,
_______________________________________________
Devel mailing list
Devel@openvz.org
https://lists.openvz.org/mailman/listinfo/devel

  parent reply	other threads:[~2026-11-11 15:01 UTC|newest]

Thread overview: 26+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-10 11:46 [Devel] [PATCH hci-8.1 v2 0/7] dm-qcow2: per-qio fast path Andrey Zhadchenko
2026-11-10 11:46 ` [Devel] [PATCH hci-8.1 v2 1/7] dm-qcow2: repeat md page lookup if it was inserted in parallel Andrey Zhadchenko
2026-11-10 11:55   ` no-reply
2026-11-10 17:56   ` Konstantin Khorenko
2026-11-10 11:46 ` [Devel] [PATCH hci-8.1 v2 2/7] dm-qcow2: wait for L2 writeback in perform_rw_mapped() Andrey Zhadchenko
2026-11-10 11:56   ` no-reply
2026-11-10 17:56   ` Konstantin Khorenko
2026-11-10 11:46 ` [Devel] [PATCH hci-8.1 v2 3/7] dm-qcow2: prepare process_one_qio() for fastpath Andrey Zhadchenko
2026-11-10 11:59   ` no-reply
2026-11-10 13:50     ` Andrey Zhadchenko
2026-11-10 17:56   ` Konstantin Khorenko
2026-11-11 14:36   ` Alexander Ivanov
2026-11-10 11:46 ` [Devel] [PATCH hci-8.1 v2 4/7] dm-qcow2: add per-qio work infrastructure Andrey Zhadchenko
2026-11-10 12:02   ` no-reply
2026-11-11 15:34     ` Andrey Zhadchenko
2026-11-10 17:56   ` Konstantin Khorenko
2026-11-10 11:46 ` [Devel] [PATCH hci-8.1 v2 5/7] dm-qcow2: implement per-qio fast processing Andrey Zhadchenko
2026-11-10 12:08   ` no-reply
2026-11-10 17:56   ` Konstantin Khorenko
2026-11-10 11:46 ` [Devel] [PATCH hci-8.1 v2 6/7] dm-qcow2: process fast path qios in per-cpu batches on a workqueue Andrey Zhadchenko
2026-11-10 12:06   ` no-reply
2026-11-10 17:56   ` Konstantin Khorenko
2026-11-11 15:00   ` Alexander Ivanov [this message]
2026-11-10 11:46 ` [Devel] [PATCH hci-8.1 v2 7/7] dm-qcow2: declare DM_TARGET_NOWAIT Andrey Zhadchenko
2026-11-10 12:06   ` no-reply
2026-11-10 17:56 ` [Devel] [PATCH hci-8.1 v2 0/7] dm-qcow2: per-qio fast path Konstantin Khorenko

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=62f1f094-a00e-4573-ba0b-2b323e73b2b9@virtuozzo.com \
    --to=alexander.ivanov@virtuozzo.com \
    --cc=andrey.zhadchenko@virtuozzo.com \
    --cc=devel@openvz.org \
    --cc=khorenko@virtuozzo.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.