[Devel] [PATCH hci-8.1 v3 2/2] selftests/ve: check memory.numa_migrate keeps the folio order

[Konstantin Khorenko <khorenko@virtuozzo.com>]

memcg_numa_migrate_new_folio() used to give every large folio a
PMD-sized destination. A smaller page cache folio then read back as
garbage, and its inode died in clear_inode() on eviction.

The test writes a file on an XFS image in 64K chunks, which gives
order-4 folios, from a cgroup bound to one NUMA node. It migrates the
cgroup to a second node with memory.numa_migrate and compares the file
with the written pattern.

Two checks make sure the test reaches the bug:

- before the migration the page cache must hold a large folio below
  the PMD size, read through /proc/self/pagemap and /proc/kpageflags;
- after it at least one such folio must sit on the target node in one
  piece, its node read through move_pages().

A failed check is a failure, not a skip, so that a setup where the
test stops reaching the bug shows up. The chunks at 2 MB aligned
offsets are not written: a folio there reads right even from an
oversized destination.

On an unfixed kernel the comparison fails and the test leaves the
image mounted, because an eviction would crash the node.

Needs root, two NUMA nodes with memory (numa=fake=2 is enough),
cgroup2 with the memory and cpuset controllers, and mkfs.xfs; skips
otherwise.

https://virtuozzo.atlassian.net/browse/VSTOR-151607

Feature: ve selftests
Signed-off-by: Konstantin Khorenko <khorenko@virtuozzo.com>
---
 tools/testing/selftests/ve/Makefile               |   1 +
 tools/testing/selftests/ve/ve_numa_migrate_test.c | 494 ++++++++++++++++++++++
 2 files changed, 495 insertions(+)

diff --git a/tools/testing/selftests/ve/Makefile b/tools/testing/selftests/ve/Makefile
index 51f9316beb99b..e96320a183844 100644
--- a/tools/testing/selftests/ve/Makefile
+++ b/tools/testing/selftests/ve/Makefile
@@ -6,6 +6,7 @@ TEST_GEN_PROGS += ve_ns_owner_test
 TEST_GEN_PROGS += ve_perms_test
 TEST_GEN_PROGS += ve_mount_accounting_test
 TEST_GEN_PROGS += ve_vtty_stress
+TEST_GEN_PROGS += ve_numa_migrate_test
 
 # CPUID faulting is x86-64 only. ARCH says "x86" for 32-bit too and lib.mk
 # picks the clang target by its emptiness, so decide by the compiler.
diff --git a/tools/testing/selftests/ve/ve_numa_migrate_test.c b/tools/testing/selftests/ve/ve_numa_migrate_test.c
new file mode 100644
index 0000000000000..2c6499195b9ba
--- /dev/null
+++ b/tools/testing/selftests/ve/ve_numa_migrate_test.c
@@ -0,0 +1,494 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * memory.numa_migrate must give a large page cache folio a destination of
+ * the same order. An oversized destination inherits the index and the
+ * reference count of the source, only the source pages are copied into it,
+ * a read of the folio is served from the destination by file offset and
+ * lands in its uninitialised part, and the inode dies in clear_inode() on
+ * eviction.
+ *
+ * Write a file on an XFS image (XFS allocates large folios for 64K writes)
+ * from a cgroup bound to one NUMA node, migrate the cgroup to another node
+ * and check that the data reads back intact.
+ *
+ * Needs root, two NUMA nodes with memory (numa=fake=2 will do), cgroup2
+ * with the memory and cpuset controllers, and mkfs.xfs. The folio sizes
+ * are read through /proc/self/pagemap and /proc/kpageflags before the
+ * migration, and the node of every page through move_pages() after it: a
+ * file without large folios below the PMD size, or a migration that
+ * moved none of them in one piece, fails the test, as it would pass
+ * without touching the bug.
+ */
+#define _GNU_SOURCE
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sched.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <sys/mount.h>
+#include <sys/mman.h>
+#include <sys/syscall.h>
+#include <linux/sched.h>
+#include <linux/kernel-page-flags.h>
+
+#include "../kselftest_harness.h"
+#include "ve_selftest.h"
+
+#define CG_NAME		"ve_numa_migrate_test"
+#define DIR_TEMPLATE	"/var/tmp/ve_numa_migrate.XXXXXX"
+/* mkfs.xfs refuses anything smaller; the image is sparse. */
+#define IMG_SIZE	(320UL << 20)
+#define FILE_SIZE	(4UL << 20)
+#define CHUNK_SIZE	(64UL << 10)
+#define PM_PFN_MASK	((1ULL << 55) - 1)
+#ifndef MADV_POPULATE_READ
+#define MADV_POPULATE_READ	22
+#endif
+#define PM_PRESENT	(1ULL << 63)
+/* The periodic memcg stats flush runs every two seconds. */
+#define STAT_RETRIES	50
+#define STAT_RETRY_US	100000
+
+FIXTURE(numa_migrate)
+{
+	int cgv2_fd;
+	int cg_fd;
+	int node_from;
+	int node_to;
+	char dir[64];
+	char img[80];
+	char mnt[80];
+	int mounted;
+	int leave_mounted;
+	int data_fd;
+};
+
+/* First two node ids of a sysfs node list such as "0-1" or "0,2-3". */
+static int two_nodes_with_memory(int *from, int *to)
+{
+	char buf[256], *tok, *save;
+	int found = 0;
+
+	if (read_file_at(AT_FDCWD, "/sys/devices/system/node/has_memory",
+			 buf, sizeof(buf)) <= 0)
+		return -1;
+
+	for (tok = strtok_r(buf, ",\n", &save); tok && found < 2;
+	     tok = strtok_r(NULL, ",\n", &save)) {
+		int a, b, n;
+
+		if (sscanf(tok, "%d-%d", &a, &b) != 2) {
+			if (sscanf(tok, "%d", &a) != 1)
+				return -1;
+			b = a;
+		}
+		for (n = a; n <= b && found < 2; n++) {
+			if (found == 0)
+				*from = n;
+			else
+				*to = n;
+			found++;
+		}
+	}
+	return found == 2 ? 0 : -1;
+}
+
+static int run_cmd(const char *fmt, ...)
+{
+	char cmd[256];
+	va_list ap;
+	int status;
+
+	va_start(ap, fmt);
+	vsnprintf(cmd, sizeof(cmd), fmt, ap);
+	va_end(ap);
+
+	status = system(cmd);
+	if (status < 0 || !WIFEXITED(status))
+		return -1;
+	return WEXITSTATUS(status);
+}
+
+/* Bytes of file pages the cgroup has on @node, from memory.numa_stat. */
+static int file_bytes_on_node(int cg_fd, int node, unsigned long long *out)
+{
+	static char buf[32768];
+	char key[16], *p;
+
+	if (read_file_at(cg_fd, "memory.numa_stat", buf, sizeof(buf)) <= 0)
+		return -1;
+	p = strstr(buf, "\nfile ");
+	if (!p)
+		return -1;
+	snprintf(key, sizeof(key), " N%d=", node);
+	p = strstr(p, key);
+	if (!p)
+		return -1;
+	*out = strtoull(p + strlen(key), NULL, 10);
+	return 0;
+}
+
+static unsigned long page_size;
+
+/* A folio at a PMD-aligned index reads right from an oversized one. */
+static bool chunk_is_hole(unsigned long off, unsigned long pmd_size)
+{
+	return !((off / CHUNK_SIZE * CHUNK_SIZE) % pmd_size);
+}
+
+#define SCAN_ORDERS	16
+
+struct folio_scan {
+	long large;	/* pages in large folios */
+	long smallest;	/* pages in the smallest large folio */
+	long on_node;	/* large folios below @pmd_pages that sit on @node */
+	long hist[2][SCAN_ORDERS];	/* folios by order, on @node / elsewhere */
+};
+
+static void folio_scan_end_run(struct folio_scan *fs, long run, int run_node,
+			       int node, long pmd_pages)
+{
+	int order = 0;
+
+	if (!run)
+		return;
+	while ((1L << order) < run && order < SCAN_ORDERS - 1)
+		order++;
+	fs->hist[run_node != node][order]++;
+	if (run == 1)
+		return;
+	if (!fs->smallest || run < fs->smallest)
+		fs->smallest = run;
+	if (run < pmd_pages && run_node == node)
+		fs->on_node++;
+}
+
+/* "64 x 64K, 3 x 4K" or "none" */
+static const char *folio_hist_str(const long *hist, char *buf, size_t len)
+{
+	size_t pos = 0;
+	int order;
+
+	for (order = SCAN_ORDERS - 1; order >= 0; order--) {
+		if (!hist[order])
+			continue;
+		pos += snprintf(buf + pos, len - pos, "%s%ld x %luK",
+				pos ? ", " : "", hist[order],
+				(page_size << order) / 1024);
+		if (pos >= len)
+			break;
+	}
+	return pos ? buf : "none";
+}
+
+/*
+ * Walk the page cache of @fd through /proc/self/pagemap, /proc/kpageflags
+ * and move_pages(2) in query mode, leaving the holes alone: faulting them
+ * in would add folios to the file. Returns -1 when that is not possible.
+ */
+static int scan_file_folios(int fd, unsigned long size, int node,
+			    unsigned long pmd_size, struct folio_scan *fs)
+{
+	unsigned long nr = size / page_size, i, off;
+	long pmd_pages = pmd_size / page_size;
+	long run = 0;
+	int run_node = -1, ret = -1;
+	int pagemap, kpageflags, *status;
+	unsigned char *map;
+	void **pages;
+	uint64_t *pm;
+
+	memset(fs, 0, sizeof(*fs));
+	pagemap = open("/proc/self/pagemap", O_RDONLY);
+	kpageflags = open("/proc/kpageflags", O_RDONLY);
+	pm = malloc(nr * sizeof(*pm));
+	pages = malloc(nr * sizeof(*pages));
+	status = malloc(nr * sizeof(*status));
+	map = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0);
+	if (pagemap < 0 || kpageflags < 0 || !pm || !pages || !status ||
+	    map == MAP_FAILED)
+		goto out;
+	for (off = 0; off < size; off += CHUNK_SIZE) {
+		if (!chunk_is_hole(off, pmd_size) &&
+		    madvise(map + off, CHUNK_SIZE, MADV_POPULATE_READ))
+			goto out;
+	}
+	if (pread(pagemap, pm, nr * sizeof(*pm),
+		  (unsigned long)map / page_size * sizeof(*pm)) !=
+	    (ssize_t)(nr * sizeof(*pm)))
+		goto out;
+	for (i = 0; i < nr; i++)
+		pages[i] = map + i * page_size;
+	if (syscall(__NR_move_pages, 0, nr, pages, NULL, status, 0))
+		goto out;
+	for (i = 0; i < nr; i++) {
+		uint64_t flags, pfn = pm[i] & PM_PFN_MASK;
+
+		if (chunk_is_hole(i * page_size, pmd_size)) {
+			folio_scan_end_run(fs, run, run_node, node, pmd_pages);
+			run = 0;
+			continue;
+		}
+		if (!(pm[i] & PM_PRESENT) || !pfn || status[i] < 0 ||
+		    pread(kpageflags, &flags, sizeof(flags),
+			  pfn * sizeof(flags)) != sizeof(flags))
+			goto out;
+		if (flags & (1ULL << KPF_COMPOUND_HEAD)) {
+			folio_scan_end_run(fs, run, run_node, node, pmd_pages);
+			fs->large++;
+			run = 1;
+			run_node = status[i];
+		} else if (flags & (1ULL << KPF_COMPOUND_TAIL)) {
+			fs->large++;
+			run++;
+		} else {
+			folio_scan_end_run(fs, run, run_node, node, pmd_pages);
+			folio_scan_end_run(fs, 1, status[i], node, pmd_pages);
+			run = 0;
+		}
+	}
+	folio_scan_end_run(fs, run, run_node, node, pmd_pages);
+	ret = 0;
+out:
+	if (map != MAP_FAILED)
+		munmap(map, size);
+	free(status);
+	free(pages);
+	free(pm);
+	if (pagemap >= 0)
+		close(pagemap);
+	if (kpageflags >= 0)
+		close(kpageflags);
+	return ret;
+}
+
+static void cleanup(FIXTURE_DATA(numa_migrate) *self)
+{
+	char path[PATH_MAX];
+
+	if (self->data_fd >= 0) {
+		close(self->data_fd);
+		self->data_fd = -1;
+	}
+	if (self->dir[0]) {
+		snprintf(path, sizeof(path), "%s/data", self->mnt);
+		unlink(path);
+		if (self->mounted)
+			umount(self->mnt);
+		unlink(self->img);
+		rmdir(self->mnt);
+		rmdir(self->dir);
+		self->dir[0] = '\0';
+	}
+	if (self->cg_fd >= 0) {
+		close(self->cg_fd);
+		self->cg_fd = -1;
+	}
+	if (self->cgv2_fd >= 0) {
+		/* The test task may still sit in the cgroup after a failure. */
+		write_u64_at(self->cgv2_fd, "cgroup.procs", getpid());
+		unlinkat(self->cgv2_fd, CG_NAME, AT_REMOVEDIR);
+		close(self->cgv2_fd);
+		self->cgv2_fd = -1;
+	}
+}
+
+/*
+ * Returns 0, or 1 with everything undone when the host cannot run the test.
+ * The harness runs no teardown after a failed setup, so the undo is here.
+ */
+static int prepare(FIXTURE_DATA(numa_migrate) *self, const char **why)
+{
+	int fd;
+
+	self->cgv2_fd = mount_cg2_fd();
+	if (self->cgv2_fd < 0) {
+		*why = "cannot mount cgroup2";
+		goto fail;
+	}
+	if (write_file_at(self->cgv2_fd, "cgroup.subtree_control",
+			  "+memory +cpuset")) {
+		*why = "cannot enable the memory and cpuset controllers";
+		goto fail;
+	}
+	if (mkdirat(self->cgv2_fd, CG_NAME, 0755) && errno != EEXIST) {
+		*why = "cannot create the test cgroup";
+		goto fail;
+	}
+	self->cg_fd = openat(self->cgv2_fd, CG_NAME, O_PATH | O_DIRECTORY);
+	if (self->cg_fd < 0) {
+		*why = "cannot open the test cgroup";
+		goto fail;
+	}
+	if (faccessat(self->cg_fd, "memory.numa_migrate", W_OK, 0)) {
+		*why = "no memory.numa_migrate";
+		goto fail;
+	}
+	if (write_u64_at(self->cg_fd, "cpuset.mems", self->node_from)) {
+		*why = "cannot bind the cgroup to a node";
+		goto fail;
+	}
+
+	snprintf(self->dir, sizeof(self->dir), DIR_TEMPLATE);
+	if (!mkdtemp(self->dir)) {
+		self->dir[0] = '\0';
+		*why = "cannot create a temporary directory";
+		goto fail;
+	}
+	snprintf(self->img, sizeof(self->img), "%s/img", self->dir);
+	snprintf(self->mnt, sizeof(self->mnt), "%s/mnt", self->dir);
+	if (mkdir(self->mnt, 0755)) {
+		*why = "cannot create the mount point";
+		goto fail;
+	}
+	fd = open(self->img, O_CREAT | O_RDWR, 0600);
+	if (fd < 0) {
+		*why = "cannot create the image file";
+		goto fail;
+	}
+	if (ftruncate(fd, IMG_SIZE)) {
+		close(fd);
+		*why = "cannot size the image file";
+		goto fail;
+	}
+	close(fd);
+	if (run_cmd("mkfs.xfs -q -f %s", self->img)) {
+		*why = "mkfs.xfs failed";
+		goto fail;
+	}
+	if (run_cmd("mount -o loop %s %s", self->img, self->mnt)) {
+		*why = "cannot loop-mount the XFS image";
+		goto fail;
+	}
+	self->mounted = 1;
+	return 0;
+fail:
+	cleanup(self);
+	return 1;
+}
+
+FIXTURE_SETUP(numa_migrate)
+{
+	const char *why;
+
+	self->cgv2_fd = -1;
+	self->cg_fd = -1;
+	self->data_fd = -1;
+	self->dir[0] = '\0';
+	page_size = sysconf(_SC_PAGESIZE);
+
+	if (geteuid())
+		SKIP(return, "needs root");
+	if (two_nodes_with_memory(&self->node_from, &self->node_to))
+		SKIP(return, "needs two NUMA nodes with memory");
+	if (prepare(self, &why))
+		SKIP(return, "%s", why);
+}
+
+FIXTURE_TEARDOWN(numa_migrate)
+{
+	if (self->leave_mounted) {
+		TH_LOG("leaving %s mounted, eviction would hit BUG_ON(nrpages)",
+		       self->mnt);
+		self->dir[0] = '\0';
+	}
+	cleanup(self);
+}
+
+TEST_F(numa_migrate, large_folio_order)
+{
+	unsigned long long on_to_before, on_to_after;
+	unsigned char *buf, *check;
+	char path[PATH_MAX], list[64];
+	unsigned long long pmd_size;
+	struct folio_scan fs;
+	char h0[128], h1[128];
+	long pmd_pages;
+	unsigned long off;
+	int fd, i;
+
+	ASSERT_EQ(write_u64_at(self->cg_fd, "cgroup.procs", getpid()), 0);
+
+	buf = malloc(CHUNK_SIZE);
+	check = malloc(CHUNK_SIZE);
+	ASSERT_NE(buf, NULL);
+	ASSERT_NE(check, NULL);
+
+	/* The old code allocated PMD-sized destinations: those were fine. */
+	if (read_u64_at(AT_FDCWD,
+			"/sys/kernel/mm/transparent_hugepage/hpage_pmd_size",
+			&pmd_size))
+		pmd_size = 2UL << 20;
+	pmd_pages = pmd_size / page_size;
+
+	snprintf(path, sizeof(path), "%s/data", self->mnt);
+	fd = open(path, O_CREAT | O_RDWR | O_TRUNC, 0600);
+	ASSERT_GE(fd, 0);
+	self->data_fd = fd;
+	for (off = 0; off < FILE_SIZE; off += CHUNK_SIZE) {
+		if (chunk_is_hole(off, pmd_size))
+			continue;
+		memset(buf, (off / CHUNK_SIZE) & 0xff, CHUNK_SIZE);
+		ASSERT_EQ(pwrite(fd, buf, CHUNK_SIZE, off), (ssize_t)CHUNK_SIZE);
+	}
+	ASSERT_EQ(fsync(fd), 0);
+	ASSERT_EQ(scan_file_folios(fd, FILE_SIZE, self->node_to, pmd_size, &fs), 0)
+		TH_LOG("cannot read the page cache folio sizes");
+	TH_LOG("before: on node %d: %s, elsewhere: %s", self->node_to,
+	       folio_hist_str(fs.hist[0], h0, sizeof(h0)),
+	       folio_hist_str(fs.hist[1], h1, sizeof(h1)));
+	ASSERT_GT(fs.large, 0)
+		TH_LOG("no large folio in the page cache, the bug is out of reach");
+	ASSERT_LT(fs.smallest, pmd_pages)
+		TH_LOG("only PMD-sized folios in the page cache, the bug is out of reach");
+
+	ASSERT_EQ(file_bytes_on_node(self->cg_fd, self->node_to,
+				     &on_to_before), 0);
+
+	/* From here on a failure means eviction would crash the kernel. */
+	self->leave_mounted = 1;
+	snprintf(list, sizeof(list), "%d,%d", self->node_from, self->node_to);
+	ASSERT_EQ(write_file_at(self->cg_fd, "cpuset.mems", list), 0);
+	ASSERT_EQ(write_u64_at(self->cg_fd, "memory.numa_migrate",
+			       self->node_to), 0);
+
+	for (off = 0; off < FILE_SIZE; off += CHUNK_SIZE) {
+		if (chunk_is_hole(off, pmd_size))
+			continue;
+		memset(buf, (off / CHUNK_SIZE) & 0xff, CHUNK_SIZE);
+		ASSERT_EQ(pread(fd, check, CHUNK_SIZE, off), (ssize_t)CHUNK_SIZE);
+		ASSERT_EQ(memcmp(buf, check, CHUNK_SIZE), 0);
+	}
+
+	/* memory.numa_stat flushes per-cpu counters lazily, give it time. */
+	for (i = 0; i < STAT_RETRIES; i++) {
+		ASSERT_EQ(file_bytes_on_node(self->cg_fd, self->node_to,
+					     &on_to_after), 0);
+		if (on_to_after >= on_to_before + FILE_SIZE)
+			break;
+		usleep(STAT_RETRY_US);
+	}
+	TH_LOG("file bytes on node %d: %llu -> %llu", self->node_to,
+	       on_to_before, on_to_after);
+
+	ASSERT_EQ(scan_file_folios(fd, FILE_SIZE, self->node_to, pmd_size, &fs), 0)
+		TH_LOG("cannot read the page cache folio sizes");
+	TH_LOG("after: on node %d: %s, elsewhere: %s", self->node_to,
+	       folio_hist_str(fs.hist[0], h0, sizeof(h0)),
+	       folio_hist_str(fs.hist[1], h1, sizeof(h1)));
+	ASSERT_GT(fs.on_node, 0)
+		TH_LOG("no folio below the PMD size reached node %d in one piece",
+		       self->node_to);
+	self->leave_mounted = 0;
+
+	free(buf);
+	free(check);
+}
+
+TEST_HARNESS_MAIN

-- 
2.43.0

_______________________________________________
Devel mailing list
Devel@openvz.org
https://lists.openvz.org/mailman/listinfo/devel