[Devel] [PATCH hci-8.1 7/7] drivers/md/dm-qcow2: process fast path qios in per-cpu batches on a workqueue

[Andrey Zhadchenko <andrey.zhadchenko@virtuozzo.com>]

queue_work() is pretty costly under high load. It takes pool->lock;
unbound pools do not track running workers (WORKER_UNBOUND is part
of WORKER_NOT_RUNNING), so an idle kworker is woken (idle exit + IPI +
context switch) even if a running worker would pick the work up right
away. PENDING is cleared when execution starts, so a qio re-queued
while its work runs wakes yet another worker just to find the collision
and go back to sleep. On a 12 cpu box with 4 submitters this shows up
as ~3 context switches per I/O, 5% of all cycles spinning on
pool->lock and 6% in the scheduler.

Batch requests with percpu llists. Additionally track if the worker
is busy and queue_work() only if it is not. Drain the whole llist in
the worker.

https://virtuozzo.atlassian.net/browse/VSTOR-95494
Feature: dm-qcow2: block device over QCOW2 files driver
Signed-off-by: Andrey Zhadchenko <andrey.zhadchenko@virtuozzo.com>
---
 drivers/md/dm-qcow2-map.c    | 99 ++++++++++++++++++++++++++++++++----
 drivers/md/dm-qcow2-target.c | 10 +++-
 drivers/md/dm-qcow2.h        |  6 ++-
 3 files changed, 102 insertions(+), 13 deletions(-)

diff --git a/drivers/md/dm-qcow2-map.c b/drivers/md/dm-qcow2-map.c
index e55186147fdf7..4fb60520b955d 100644
--- a/drivers/md/dm-qcow2-map.c
+++ b/drivers/md/dm-qcow2-map.c
@@ -87,6 +87,7 @@ static void process_read_qio(struct qcow2 *qcow2, struct qio *qio,
 			     struct qcow2_map *map);
 static void init_qrq_and_embedded_qio(struct qcow2_target *tgt, struct request *rq,
 				      struct qcow2_rq *qrq, struct qio *qio);
+static void qcow2_fast_submit(struct qcow2_target *tgt, struct qio *qio);
 
 static loff_t bytes_off_in_cluster(struct qcow2 *qcow2, struct qio *qio)
 {
@@ -362,8 +363,6 @@ struct qio *qcow2_alloc_qio(mempool_t *pool, bool zero)
 }
 ALLOW_ERROR_INJECTION(qcow2_alloc_qio, NULL);
 
-static void do_qio_work(struct work_struct *ws);
-
 void qcow2_init_qio(struct qio *qio, unsigned int bi_op, struct qcow2 *qcow2)
 {
 	qio->qcow2 = qcow2;
@@ -376,7 +375,6 @@ void qcow2_init_qio(struct qio *qio, unsigned int bi_op, struct qcow2 *qcow2)
 	qio->ref_index = REF_INDEX_INVALID;
 	qio->state = QIO_STATE_START;
 	atomic_set(&qio->remaining, 1);
-	INIT_WORK(&qio->worker, do_qio_work);
 
 	/*
 	 * Initially set into BLK_STS_OK, while aio complete,
@@ -4483,7 +4481,7 @@ static void qcow2_submit_embedded_qio(struct qcow2_target *tgt, struct qio *qio)
 
 	if (blk_rq_bytes(rq)) {
 		queue_list_id = QLIST_EMBEDDED;
-		worker = &qio->worker;
+		worker = NULL;
 	} else {
 		WARN_ON_ONCE(qio->bi_op != REQ_OP_FLUSH);
 		queue_list_id = QLIST_FLUSH;
@@ -4506,8 +4504,13 @@ static void qcow2_submit_embedded_qio(struct qcow2_target *tgt, struct qio *qio)
 	}
 	spin_unlock_irqrestore(&qcow2->deferred_lock, flags);
 
-	if (queue)
+	if (!queue)
+		return;
+
+	if (worker)
 		queue_work(tgt->wq, worker);
+	else
+		qcow2_fast_submit(tgt, qio);
 }
 
 void qcow2_submit_embedded_qios(struct qcow2_target *tgt, struct list_head *list)
@@ -4602,15 +4605,11 @@ static void qio_try_fast(struct qio *qio, struct list_head *deferred_qios)
 		list_add(&qio->link, deferred_qios);
 }
 
-static void do_qio_work(struct work_struct *ws)
+static void do_qio_work(struct qio *qio)
 {
-	struct qio *qio = container_of(ws, struct qio, worker);
-	struct qio *i, *next;
 	struct qcow2 *qcow2 = qio->qcow2;
 	struct qcow2_target *tgt = qcow2->tgt;
-	unsigned int pflags = current->flags;
-
-	current->flags |= PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO;
+	struct qio *i, *next;
 
 	switch (qio->state) {
 	case QIO_STATE_START: {
@@ -4636,10 +4635,88 @@ static void do_qio_work(struct work_struct *ws)
 		qio_endio(qio);
 		break;
 	}
+}
+
+struct qcow2_fast_cpu {
+	struct llist_head qios;
+	struct work_struct work;
+	atomic_t busy;
+};
+
+static void qcow2_fast_submit(struct qcow2_target *tgt, struct qio *qio)
+{
+	struct qcow2_fast_cpu *fc = get_cpu_ptr(tgt->fast);
+
+	llist_add(&qio->fast_link, &fc->qios);
+	/* Pairs with smp_mb() in qcow2_fast_work() */
+	smp_mb();
+	if (!atomic_read(&fc->busy) && atomic_cmpxchg(&fc->busy, 0, 1) == 0)
+		queue_work(tgt->wq, &fc->work);
+	put_cpu_ptr(tgt->fast);
+}
+
+static void qcow2_fast_work(struct work_struct *work)
+{
+	struct qcow2_fast_cpu *fc = container_of(work, struct qcow2_fast_cpu, work);
+	unsigned int pflags = current->flags;
+	struct llist_node *node;
+	struct qio *qio, *next;
+
+	current->flags |= PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO;
+
+	for (;;) {
+		node = llist_del_all(&fc->qios);
+		if (!node) {
+			atomic_set(&fc->busy, 0);
+			/* just to be sure we look at the list after busy is posted */
+			smp_mb();
+			if (llist_empty(&fc->qios))
+				break;
+			if (atomic_cmpxchg(&fc->busy, 0, 1) != 0)
+				break;
+			continue;
+		}
+		node = llist_reverse_order(node);
+		llist_for_each_entry_safe(qio, next, node, fast_link)
+			do_qio_work(qio);
+	}
 
 	current_restore_flags(pflags, PF_LOCAL_THROTTLE|PF_MEMALLOC_NOIO);
 }
 
+void qcow2_fast_exit(struct qcow2_target *tgt)
+{
+	struct qcow2_fast_cpu *fc;
+	int cpu;
+
+	if (!tgt->fast)
+		return;
+	for_each_possible_cpu(cpu) {
+		fc = per_cpu_ptr(tgt->fast, cpu);
+		WARN_ON_ONCE(!llist_empty(&fc->qios));
+	}
+	free_percpu(tgt->fast);
+	tgt->fast = NULL;
+}
+
+int qcow2_fast_init(struct qcow2_target *tgt)
+{
+	struct qcow2_fast_cpu *fc;
+	int cpu;
+
+	tgt->fast = alloc_percpu(struct qcow2_fast_cpu);
+	if (!tgt->fast)
+		return -ENOMEM;
+	for_each_possible_cpu(cpu) {
+		fc = per_cpu_ptr(tgt->fast, cpu);
+		init_llist_head(&fc->qios);
+		INIT_WORK(&fc->work, qcow2_fast_work);
+		atomic_set(&fc->busy, 0);
+	}
+
+	return 0;
+}
+
 struct qio_data_llseek_hole {
 	struct completion compl;
 	struct qio *higher;
diff --git a/drivers/md/dm-qcow2-target.c b/drivers/md/dm-qcow2-target.c
index 4b9ae86ac9443..290409db008b8 100644
--- a/drivers/md/dm-qcow2-target.c
+++ b/drivers/md/dm-qcow2-target.c
@@ -253,6 +253,7 @@ static void qcow2_tgt_destroy(struct qcow2_target *tgt)
 		/* Now kill the queue */
 		destroy_workqueue(tgt->wq);
 	}
+	qcow2_fast_exit(tgt);
 	qcow2_merge_backward_cancel(tgt);
 
 	mempool_destroy(tgt->qio_pool);
@@ -477,6 +478,11 @@ static struct qcow2_target *alloc_qcow2_target(struct dm_target *ti)
 		goto out_pool;
 	}
 
+	if (qcow2_fast_init(tgt)) {
+		ti->error = "Can't alloc fast path slots";
+		goto out_wq;
+	}
+
 	for (i = 0; i < 2; i++) {
 		release = i ? inflight_ref_exit1 : inflight_ref_exit0;
 		if (percpu_ref_init(&tgt->inflight_ref[i], release,
@@ -484,7 +490,7 @@ static struct qcow2_target *alloc_qcow2_target(struct dm_target *ti)
 			if (i)
 				percpu_ref_exit(&tgt->inflight_ref[0]);
 			ti->error = "could not alloc percpu_ref";
-			goto out_wq;
+			goto out_fast;
 		}
 	}
 
@@ -503,6 +509,8 @@ static struct qcow2_target *alloc_qcow2_target(struct dm_target *ti)
 	qcow2_set_service_operations(ti, false);
 
 	return tgt;
+out_fast:
+	qcow2_fast_exit(tgt);
 out_wq:
 	destroy_workqueue(tgt->wq);
 out_pool:
diff --git a/drivers/md/dm-qcow2.h b/drivers/md/dm-qcow2.h
index 003e01a4d3ec9..6e2ac8d9979fd 100644
--- a/drivers/md/dm-qcow2.h
+++ b/drivers/md/dm-qcow2.h
@@ -4,6 +4,7 @@
 
 #include <linux/percpu-refcount.h>
 #include <linux/device-mapper.h>
+#include <linux/llist.h>
 #include <linux/fs.h>
 #include <linux/eventfd.h>
 
@@ -192,6 +193,7 @@ struct qcow2_target {
 	 */
 	struct qcow2 *top;
 	struct workqueue_struct *wq;
+	struct qcow2_fast_cpu __percpu *fast;
 
 	struct completion inflight_ref_comp;
 	struct percpu_ref inflight_ref[2];
@@ -373,7 +375,7 @@ struct qio {
 	struct qcow2 *qcow2;
 	qcow2_endio_t endio_cb;
 	void *endio_cb_data;
-	struct work_struct worker;
+	struct llist_node fast_link;
 	int state;
 };
 
@@ -390,6 +392,8 @@ loff_t qcow2_llseek_hole(struct dm_target *ti, loff_t offset, int whence);
 
 void do_qcow2_work(struct work_struct *ws);
 void do_qcow2_fsync_work(struct work_struct *ws);
+int qcow2_fast_init(struct qcow2_target *tgt);
+void qcow2_fast_exit(struct qcow2_target *tgt);
 int qcow2_alloc_and_insert_md_page(struct qcow2 *qcow2, u64 index,
 					struct md_page **md);
 struct md_page *qcow2_md_page_find_or_postpone(struct qcow2 *qcow2,
-- 
2.43.5

_______________________________________________
Devel mailing list
Devel@openvz.org
https://lists.openvz.org/mailman/listinfo/devel