[Devel] [PATCH hci-8.1 v4 2/2] selftests/ve: check memory.numa_migrate keeps the folio order

[Konstantin Khorenko <khorenko@virtuozzo.com>]

memcg_numa_migrate_new_folio() used to give every large folio a
PMD-sized destination. A smaller page cache folio then read back as
garbage, and its inode died in clear_inode() on eviction.

The test writes a file on an XFS image in 64K chunks, which gives
order-4 folios, from a cgroup bound to one NUMA node. It widens the
cgroup's cpuset.mems to both nodes, as the destination allocation obeys
the writer's cpuset, migrates the cgroup to the second node with
memory.numa_migrate and compares the file with the written pattern.
The comparison is what catches the bug.

The chunks at 2 MB aligned offsets are not written: a folio there
reads right even from an oversized destination.

The folios are tracked page by page through /proc/self/pagemap,
/proc/kpageflags and move_pages(), so that a pass means the large folio
path really ran:

- before the migration the page cache must hold a large folio below
  the PMD size;
- after it every such folio must have kept its order on the target
  node, been split into base pages or stayed behind, and at least one
  must have kept its order.

A failed check is a failure, not a skip, so that a setup where the
test stops reaching the bug shows up. A fragmented target node fails
it too, as the large destinations are allocated without reclaim.

On an unfixed release kernel the test fails on the comparison, or on
the folio checks when the target node has no free 2 MB block. A debug
kernel that gets an order-9 destination dies earlier, on the
VM_BUG_ON_FOLIO() in mem_cgroup_migrate().

Needs root, two NUMA nodes with memory (numa=fake=2 is enough),
cgroup2 with the memory and cpuset controllers and memory.numa_migrate,
and mkfs.xfs with loop devices; skips otherwise.

https://virtuozzo.atlassian.net/browse/VSTOR-151607

Feature: ve selftests
Signed-off-by: Konstantin Khorenko <khorenko@virtuozzo.com>
---
 tools/testing/selftests/ve/.gitignore             |   1 +
 tools/testing/selftests/ve/Makefile               |   1 +
 tools/testing/selftests/ve/ve_numa_migrate_test.c | 562 ++++++++++++++++++++++
 3 files changed, 564 insertions(+)

diff --git a/tools/testing/selftests/ve/.gitignore b/tools/testing/selftests/ve/.gitignore
index 8ab1bfe6f5e4b..3d1179bd2c5f4 100644
--- a/tools/testing/selftests/ve/.gitignore
+++ b/tools/testing/selftests/ve/.gitignore
@@ -3,3 +3,4 @@ ve_perms_test
 ve_mount_accounting_test
 ve_cpuid_override_test
 ve_vtty_stress
+ve_numa_migrate_test
diff --git a/tools/testing/selftests/ve/Makefile b/tools/testing/selftests/ve/Makefile
index 51f9316beb99b..e96320a183844 100644
--- a/tools/testing/selftests/ve/Makefile
+++ b/tools/testing/selftests/ve/Makefile
@@ -6,6 +6,7 @@ TEST_GEN_PROGS += ve_ns_owner_test
 TEST_GEN_PROGS += ve_perms_test
 TEST_GEN_PROGS += ve_mount_accounting_test
 TEST_GEN_PROGS += ve_vtty_stress
+TEST_GEN_PROGS += ve_numa_migrate_test
 
 # CPUID faulting is x86-64 only. ARCH says "x86" for 32-bit too and lib.mk
 # picks the clang target by its emptiness, so decide by the compiler.
diff --git a/tools/testing/selftests/ve/ve_numa_migrate_test.c b/tools/testing/selftests/ve/ve_numa_migrate_test.c
new file mode 100644
index 0000000000000..02e134bc914d4
--- /dev/null
+++ b/tools/testing/selftests/ve/ve_numa_migrate_test.c
@@ -0,0 +1,562 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * memory.numa_migrate must give a large page cache folio a destination of
+ * the same order. An oversized destination inherits the index and the
+ * reference count of the source, only the source pages are copied into it,
+ * a read of the folio is served from the destination by file offset and
+ * lands in its uninitialised part, and the inode dies in clear_inode() on
+ * eviction.
+ *
+ * Write a file on an XFS image (XFS allocates large folios for 64K writes)
+ * from a cgroup bound to one NUMA node, migrate the cgroup to another node
+ * and check that the data reads back intact.
+ *
+ * Needs root, two NUMA nodes with memory (numa=fake=2 will do), cgroup2
+ * with the memory and cpuset controllers and memory.numa_migrate, and
+ * mkfs.xfs with loop devices. Before and after the migration the folio of
+ * every page is read through /proc/self/pagemap and /proc/kpageflags and
+ * its node through move_pages(): a file without large folios below the PMD
+ * size, or a migration that moved none of them in one piece, fails the
+ * test, as it would pass without touching the bug.
+ */
+#define _GNU_SOURCE
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sched.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <sys/mount.h>
+#include <sys/mman.h>
+#include <sys/syscall.h>
+#include <linux/kernel-page-flags.h>
+
+#include "../kselftest_harness.h"
+#include "ve_selftest.h"
+
+#define CG_NAME		"ve_numa_migrate_test"
+#define DIR_TEMPLATE	"/var/tmp/ve_numa_migrate.XXXXXX"
+/* mkfs.xfs refuses anything smaller; the image is sparse. */
+#define IMG_SIZE	(320UL << 20)
+#define FILE_SIZE	(4UL << 20)
+#define CHUNK_SIZE	(64UL << 10)
+#define PM_PFN_MASK	((1ULL << 55) - 1)
+#define PM_PRESENT	(1ULL << 63)
+#ifndef MADV_POPULATE_READ
+#define MADV_POPULATE_READ	22
+#endif
+#define SCAN_ORDERS	16
+/* The periodic memcg stats flush runs every two seconds. */
+#define STAT_RETRIES	50
+#define STAT_RETRY_US	100000
+
+FIXTURE(numa_migrate)
+{
+	int cgv2_fd;
+	int cg_fd;
+	int node_from;
+	int node_to;
+	char dir[64];
+	char img[80];
+	char mnt[80];
+	int mounted;
+	int data_fd;
+};
+
+/* First two node ids of a sysfs node list such as "0-1" or "0,2-3". */
+static int two_nodes_with_memory(int *from, int *to)
+{
+	char buf[256], *tok, *save;
+	int found = 0;
+
+	if (read_file_at(AT_FDCWD, "/sys/devices/system/node/has_memory",
+			 buf, sizeof(buf)) <= 0)
+		return -1;
+
+	for (tok = strtok_r(buf, ",\n", &save); tok && found < 2;
+	     tok = strtok_r(NULL, ",\n", &save)) {
+		int a, b, n;
+
+		if (sscanf(tok, "%d-%d", &a, &b) != 2) {
+			if (sscanf(tok, "%d", &a) != 1)
+				return -1;
+			b = a;
+		}
+		for (n = a; n <= b && found < 2; n++) {
+			if (found == 0)
+				*from = n;
+			else
+				*to = n;
+			found++;
+		}
+	}
+	return found == 2 ? 0 : -1;
+}
+
+static int run_cmd(const char *fmt, ...)
+{
+	char cmd[256];
+	va_list ap;
+	int status;
+
+	va_start(ap, fmt);
+	vsnprintf(cmd, sizeof(cmd), fmt, ap);
+	va_end(ap);
+
+	status = system(cmd);
+	if (status < 0 || !WIFEXITED(status))
+		return -1;
+	return WEXITSTATUS(status);
+}
+
+/* Bytes of file pages the cgroup has on @node, from memory.numa_stat. */
+static int file_bytes_on_node(int cg_fd, int node, unsigned long long *out)
+{
+	static char buf[32768];
+	char key[16], *p;
+
+	if (read_file_at(cg_fd, "memory.numa_stat", buf, sizeof(buf)) <= 0)
+		return -1;
+	p = strstr(buf, "\nfile ");
+	if (!p)
+		return -1;
+	snprintf(key, sizeof(key), " N%d=", node);
+	p = strstr(p, key);
+	if (!p)
+		return -1;
+	*out = strtoull(p + strlen(key), NULL, 10);
+	return 0;
+}
+
+static unsigned long page_size;
+
+/* A folio at a PMD-aligned index reads right from an oversized one. */
+static bool chunk_is_hole(unsigned long off, unsigned long pmd_size)
+{
+	return !((off / CHUNK_SIZE * CHUNK_SIZE) % pmd_size);
+}
+
+/* One per page of the file: the node it sits on and the folio around it. */
+struct page_info {
+	int node;
+	int order;	/* -1 in a hole */
+	bool head;
+};
+
+static void fill_run(struct page_info *pi, unsigned long start, long run,
+		     int node)
+{
+	int order = 0;
+	long i;
+
+	while ((1L << order) < run)
+		order++;
+	for (i = 0; i < run; i++) {
+		pi[start + i].node = node;
+		pi[start + i].order = order;
+		pi[start + i].head = !i;
+	}
+}
+
+/*
+ * Walk the page cache of @fd through /proc/self/pagemap, /proc/kpageflags
+ * and move_pages(2) in query mode, leaving the holes alone: faulting them
+ * in would add folios to the file. Returns -1 when that is not possible.
+ */
+static int scan_file_folios(int fd, unsigned long size, unsigned long pmd_size,
+			    struct page_info *pi)
+{
+	unsigned long nr = size / page_size, i, off, start = 0;
+	long run = 0;
+	int run_node = -1, ret = -1;
+	int pagemap, kpageflags, *status;
+	unsigned char *map;
+	void **pages;
+	uint64_t *pm;
+
+	for (i = 0; i < nr; i++)
+		pi[i].order = -1;
+	pagemap = open("/proc/self/pagemap", O_RDONLY);
+	kpageflags = open("/proc/kpageflags", O_RDONLY);
+	pm = malloc(nr * sizeof(*pm));
+	pages = malloc(nr * sizeof(*pages));
+	status = malloc(nr * sizeof(*status));
+	map = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0);
+	if (pagemap < 0 || kpageflags < 0 || !pm || !pages || !status ||
+	    map == MAP_FAILED)
+		goto out;
+	for (off = 0; off < size; off += CHUNK_SIZE) {
+		if (!chunk_is_hole(off, pmd_size) &&
+		    madvise(map + off, CHUNK_SIZE, MADV_POPULATE_READ))
+			goto out;
+	}
+	if (pread(pagemap, pm, nr * sizeof(*pm),
+		  (unsigned long)map / page_size * sizeof(*pm)) !=
+	    (ssize_t)(nr * sizeof(*pm)))
+		goto out;
+	for (i = 0; i < nr; i++)
+		pages[i] = map + i * page_size;
+	if (syscall(__NR_move_pages, 0, nr, pages, NULL, status, 0))
+		goto out;
+	for (i = 0; i < nr; i++) {
+		uint64_t flags, pfn = pm[i] & PM_PFN_MASK;
+
+		if (chunk_is_hole(i * page_size, pmd_size)) {
+			fill_run(pi, start, run, run_node);
+			run = 0;
+			continue;
+		}
+		if (!(pm[i] & PM_PRESENT) || !pfn || status[i] < 0 ||
+		    pread(kpageflags, &flags, sizeof(flags),
+			  pfn * sizeof(flags)) != sizeof(flags))
+			goto out;
+		if ((flags & (1ULL << KPF_COMPOUND_TAIL)) && run) {
+			run++;
+			continue;
+		}
+		fill_run(pi, start, run, run_node);
+		start = i;
+		run = 1;
+		run_node = status[i];
+		if (!(flags & (1ULL << KPF_COMPOUND_HEAD))) {
+			fill_run(pi, start, run, run_node);
+			run = 0;
+		}
+	}
+	fill_run(pi, start, run, run_node);
+	ret = 0;
+out:
+	if (map != MAP_FAILED)
+		munmap(map, size);
+	free(status);
+	free(pages);
+	free(pm);
+	if (pagemap >= 0)
+		close(pagemap);
+	if (kpageflags >= 0)
+		close(kpageflags);
+	return ret;
+}
+
+/* The folio sizes of a scan, "62 x 64K, 3 x 4K", or "none". */
+static const char *folio_sizes_str(const struct page_info *pi,
+				   unsigned long nr, char *buf, size_t len)
+{
+	long hist[SCAN_ORDERS] = {};
+	size_t pos = 0;
+	unsigned long i;
+	int order;
+
+	for (i = 0; i < nr; i++) {
+		if (pi[i].order >= 0 && pi[i].head)
+			hist[pi[i].order < SCAN_ORDERS ? pi[i].order :
+			     SCAN_ORDERS - 1]++;
+	}
+	for (order = SCAN_ORDERS - 1; order >= 0; order--) {
+		if (!hist[order])
+			continue;
+		pos += snprintf(buf + pos, len - pos, "%s%ld x %luK",
+				pos ? ", " : "", hist[order],
+				(page_size << order) / 1024);
+		if (pos >= len)
+			break;
+	}
+	return pos ? buf : "none";
+}
+
+/* Large folios below the PMD size in a scan. */
+static long large_below_pmd(const struct page_info *pi, unsigned long nr,
+			    long pmd_pages)
+{
+	unsigned long i;
+	long count = 0;
+
+	for (i = 0; i < nr; i++) {
+		if (pi[i].head && pi[i].order > 0 &&
+		    (1L << pi[i].order) < pmd_pages)
+			count++;
+	}
+	return count;
+}
+
+/* What became of the large folios below the PMD size of @before. */
+struct migrated {
+	long intact;	/* the same folio, now on the target node */
+	long split;	/* its pages on the target node as base pages */
+	long left;	/* still on the source node */
+	long wrong;	/* another order, or moved in part */
+};
+
+static void compare_folios(const struct page_info *before,
+			   const struct page_info *after, unsigned long nr,
+			   int node, long pmd_pages, struct migrated *m)
+{
+	unsigned long i, j;
+
+	memset(m, 0, sizeof(*m));
+	for (i = 0; i < nr; i++) {
+		long pages, on_node = 0, base = 0;
+
+		if (before[i].order <= 0 || !before[i].head)
+			continue;
+		pages = 1L << before[i].order;
+		if (pages >= pmd_pages)
+			continue;
+		for (j = i; j < i + pages && j < nr; j++) {
+			on_node += after[j].node == node;
+			base += !after[j].order;
+		}
+		if (!on_node)
+			m->left++;
+		else if (on_node != pages)
+			m->wrong++;
+		else if (after[i].head && after[i].order == before[i].order)
+			m->intact++;
+		else if (base == pages)
+			m->split++;
+		else
+			m->wrong++;
+	}
+}
+
+static void cleanup(FIXTURE_DATA(numa_migrate) *self)
+{
+	char path[PATH_MAX];
+
+	if (self->data_fd >= 0) {
+		close(self->data_fd);
+		self->data_fd = -1;
+	}
+	if (self->dir[0]) {
+		snprintf(path, sizeof(path), "%s/data", self->mnt);
+		unlink(path);
+		if (self->mounted)
+			umount(self->mnt);
+		unlink(self->img);
+		rmdir(self->mnt);
+		rmdir(self->dir);
+		self->dir[0] = '\0';
+	}
+	if (self->cg_fd >= 0) {
+		close(self->cg_fd);
+		self->cg_fd = -1;
+	}
+	if (self->cgv2_fd >= 0) {
+		/* The test task may still sit in the cgroup after a failure. */
+		write_u64_at(self->cgv2_fd, "cgroup.procs", getpid());
+		unlinkat(self->cgv2_fd, CG_NAME, AT_REMOVEDIR);
+		close(self->cgv2_fd);
+		self->cgv2_fd = -1;
+	}
+}
+
+/*
+ * Returns 0, or 1 with everything undone when the host cannot run the test.
+ * The harness runs no teardown after a failed setup, so the undo is here.
+ */
+static int prepare(FIXTURE_DATA(numa_migrate) *self, const char **why)
+{
+	int fd;
+
+	self->cgv2_fd = mount_cg2_fd();
+	if (self->cgv2_fd < 0) {
+		*why = "cannot mount cgroup2";
+		goto fail;
+	}
+	if (write_file_at(self->cgv2_fd, "cgroup.subtree_control",
+			  "+memory +cpuset")) {
+		*why = "cannot enable the memory and cpuset controllers";
+		goto fail;
+	}
+	if (mkdirat(self->cgv2_fd, CG_NAME, 0755) && errno != EEXIST) {
+		*why = "cannot create the test cgroup";
+		goto fail;
+	}
+	self->cg_fd = openat(self->cgv2_fd, CG_NAME, O_PATH | O_DIRECTORY);
+	if (self->cg_fd < 0) {
+		*why = "cannot open the test cgroup";
+		goto fail;
+	}
+	if (faccessat(self->cg_fd, "memory.numa_migrate", W_OK, 0)) {
+		*why = "no memory.numa_migrate";
+		goto fail;
+	}
+	if (write_u64_at(self->cg_fd, "cpuset.mems", self->node_from)) {
+		*why = "cannot bind the cgroup to a node";
+		goto fail;
+	}
+
+	snprintf(self->dir, sizeof(self->dir), DIR_TEMPLATE);
+	if (!mkdtemp(self->dir)) {
+		self->dir[0] = '\0';
+		*why = "cannot create a temporary directory";
+		goto fail;
+	}
+	snprintf(self->img, sizeof(self->img), "%s/img", self->dir);
+	snprintf(self->mnt, sizeof(self->mnt), "%s/mnt", self->dir);
+	if (mkdir(self->mnt, 0755)) {
+		*why = "cannot create the mount point";
+		goto fail;
+	}
+	fd = open(self->img, O_CREAT | O_RDWR, 0600);
+	if (fd < 0) {
+		*why = "cannot create the image file";
+		goto fail;
+	}
+	if (ftruncate(fd, IMG_SIZE)) {
+		close(fd);
+		*why = "cannot size the image file";
+		goto fail;
+	}
+	close(fd);
+	if (run_cmd("mkfs.xfs -q -f %s", self->img)) {
+		*why = "mkfs.xfs failed";
+		goto fail;
+	}
+	if (run_cmd("mount -o loop %s %s", self->img, self->mnt)) {
+		*why = "cannot loop-mount the XFS image";
+		goto fail;
+	}
+	self->mounted = 1;
+	return 0;
+fail:
+	cleanup(self);
+	return 1;
+}
+
+FIXTURE_SETUP(numa_migrate)
+{
+	const char *why;
+
+	self->cgv2_fd = -1;
+	self->cg_fd = -1;
+	self->data_fd = -1;
+	self->dir[0] = '\0';
+	page_size = sysconf(_SC_PAGESIZE);
+
+	if (geteuid())
+		SKIP(return, "needs root");
+	if (two_nodes_with_memory(&self->node_from, &self->node_to))
+		SKIP(return, "needs two NUMA nodes with memory");
+	if (prepare(self, &why))
+		SKIP(return, "%s", why);
+}
+
+FIXTURE_TEARDOWN(numa_migrate)
+{
+	cleanup(self);
+}
+
+TEST_F(numa_migrate, large_folio_order)
+{
+	unsigned long long on_to_before, on_to_after;
+	unsigned char *buf, *check;
+	char path[PATH_MAX], list[64];
+	unsigned long long pmd_size, written = 0;
+	struct page_info *before, *after;
+	struct migrated m;
+	char sizes[128];
+	long pmd_pages;
+	unsigned long off;
+	int fd, i;
+
+	ASSERT_EQ(write_u64_at(self->cg_fd, "cgroup.procs", getpid()), 0);
+
+	buf = malloc(CHUNK_SIZE);
+	check = malloc(CHUNK_SIZE);
+	before = calloc(FILE_SIZE / page_size, sizeof(*before));
+	after = calloc(FILE_SIZE / page_size, sizeof(*after));
+	ASSERT_NE(buf, NULL);
+	ASSERT_NE(check, NULL);
+	ASSERT_NE(before, NULL);
+	ASSERT_NE(after, NULL);
+
+	/* PMD-sized folios got a correct destination from the old code too. */
+	if (read_u64_at(AT_FDCWD,
+			"/sys/kernel/mm/transparent_hugepage/hpage_pmd_size",
+			&pmd_size))
+		pmd_size = 2UL << 20;
+	pmd_pages = pmd_size / page_size;
+
+	snprintf(path, sizeof(path), "%s/data", self->mnt);
+	fd = open(path, O_CREAT | O_RDWR | O_TRUNC, 0600);
+	ASSERT_GE(fd, 0);
+	self->data_fd = fd;
+	for (off = 0; off < FILE_SIZE; off += CHUNK_SIZE) {
+		if (chunk_is_hole(off, pmd_size))
+			continue;
+		memset(buf, (off / CHUNK_SIZE) & 0xff, CHUNK_SIZE);
+		ASSERT_EQ(pwrite(fd, buf, CHUNK_SIZE, off), (ssize_t)CHUNK_SIZE);
+		written += CHUNK_SIZE;
+	}
+	ASSERT_EQ(fsync(fd), 0);
+	ASSERT_EQ(scan_file_folios(fd, FILE_SIZE, pmd_size, before), 0)
+		TH_LOG("cannot read the page cache folio sizes");
+	TH_LOG("before: %s", folio_sizes_str(before, FILE_SIZE / page_size,
+					     sizes, sizeof(sizes)));
+	ASSERT_GT(large_below_pmd(before, FILE_SIZE / page_size, pmd_pages), 0)
+		TH_LOG("no large folio below the PMD size, the bug is out of reach");
+
+	ASSERT_EQ(file_bytes_on_node(self->cg_fd, self->node_to,
+				     &on_to_before), 0);
+
+	snprintf(list, sizeof(list), "%d,%d", self->node_from, self->node_to);
+	ASSERT_EQ(write_file_at(self->cg_fd, "cpuset.mems", list), 0);
+
+	/*
+	 * With a scan limit the kernel walks the LRUs in batches until a pass
+	 * finds nothing, instead of sizing the walk from the LRU stats, which
+	 * lag on the classic LRU; the limit counts folios scanned, far above
+	 * the need.
+	 */
+	snprintf(list, sizeof(list), "%d %lu", self->node_to,
+		 IMG_SIZE / page_size);
+	ASSERT_EQ(write_file_at(self->cg_fd, "memory.numa_migrate", list), 0)
+		TH_LOG("memory.numa_migrate: %s", strerror(errno));
+
+	for (off = 0; off < FILE_SIZE; off += CHUNK_SIZE) {
+		if (chunk_is_hole(off, pmd_size))
+			continue;
+		memset(buf, (off / CHUNK_SIZE) & 0xff, CHUNK_SIZE);
+		ASSERT_EQ(pread(fd, check, CHUNK_SIZE, off), (ssize_t)CHUNK_SIZE);
+		ASSERT_EQ(memcmp(buf, check, CHUNK_SIZE), 0)
+			TH_LOG("the chunk at %lu reads back wrong", off);
+	}
+
+	/* memory.numa_stat flushes per-cpu counters lazily, give it time. */
+	for (i = 0; i < STAT_RETRIES; i++) {
+		ASSERT_EQ(file_bytes_on_node(self->cg_fd, self->node_to,
+					     &on_to_after), 0);
+		if (on_to_after >= on_to_before + written)
+			break;
+		usleep(STAT_RETRY_US);
+	}
+	TH_LOG("file bytes on node %d: %llu -> %llu", self->node_to,
+	       on_to_before, on_to_after);
+
+	ASSERT_EQ(scan_file_folios(fd, FILE_SIZE, pmd_size, after), 0)
+		TH_LOG("cannot read the page cache folio sizes");
+	compare_folios(before, after, FILE_SIZE / page_size, self->node_to,
+		       pmd_pages, &m);
+	TH_LOG("after: %s; %ld intact, %ld split, %ld left, %ld wrong",
+	       folio_sizes_str(after, FILE_SIZE / page_size, sizes,
+			       sizeof(sizes)),
+	       m.intact, m.split, m.left, m.wrong);
+	ASSERT_EQ(m.wrong, 0)
+		TH_LOG("a folio changed its order or moved in part (memory pressure?)");
+	ASSERT_GT(m.intact, 0)
+		TH_LOG("no folio below the PMD size reached node %d in one piece (fragmented?)",
+		       self->node_to);
+
+	free(after);
+	free(before);
+	free(buf);
+	free(check);
+}
+
+TEST_HARNESS_MAIN

-- 
2.43.0

_______________________________________________
Devel mailing list
Devel@openvz.org
https://lists.openvz.org/mailman/listinfo/devel