[Devel] [PATCH vz10 v2 2/2] selftests: ve: introduce vtty pair teardown stress test

[Eva Kurchatova <eva.kurchatova@virtuozzo.com>]

A vtty pair is created by the first master open and torn down by the
last close. The close racing the peer's close used to let both sides
decide they were the final one and call release_tty() twice, and a
master opening again while a teardown was in flight used to hand out a
reference to a tty_struct about to be freed.

Hammer both ends. The master comes from the vzctl ioctl, which does
not look the VE up. The slave is only reachable from inside the VE, as
TTY_MAJOR minor 1..MAX_NR_VTTY_CONSOLES, which vtty_driver() maps to
vttys index minor - 1, so the test builds a VE with cgroup2, sets
ve.veid to the cgroup it made, and runs the slave side there.

One of the symtoms of a buggy kernel is dmesg warning, not just an
eventual crash, so the test checks that. A crashed node should also
report test failure in CI, but that's less verbose, hence the check.

https://virtuozzo.atlassian.net/browse/VSTOR-144771

Feature: ve selftest
Signed-off-by: Eva Kurchatova <eva.kurchatova@virtuozzo.com>
---
 tools/testing/selftests/ve/.gitignore       |   1 +
 tools/testing/selftests/ve/Makefile         |   1 +
 tools/testing/selftests/ve/settings         |   1 +
 tools/testing/selftests/ve/ve_vtty_stress.c | 250 ++++++++++++++++++++
 4 files changed, 253 insertions(+)
 create mode 100644 tools/testing/selftests/ve/settings
 create mode 100644 tools/testing/selftests/ve/ve_vtty_stress.c

diff --git a/tools/testing/selftests/ve/.gitignore b/tools/testing/selftests/ve/.gitignore
index afa4c568c2c9..9d8d86093827 100644
--- a/tools/testing/selftests/ve/.gitignore
+++ b/tools/testing/selftests/ve/.gitignore
@@ -1,2 +1,3 @@
 ve_ns_owner_test
 ve_perms_test
+ve_vtty_stress
diff --git a/tools/testing/selftests/ve/Makefile b/tools/testing/selftests/ve/Makefile
index ec40cbc7b3a1..6cd5823c4d93 100644
--- a/tools/testing/selftests/ve/Makefile
+++ b/tools/testing/selftests/ve/Makefile
@@ -4,5 +4,6 @@ CFLAGS += -g -Wall -O2
 
 TEST_GEN_PROGS += ve_ns_owner_test
 TEST_GEN_PROGS += ve_perms_test
+TEST_GEN_PROGS += ve_vtty_stress
 
 include ../lib.mk
diff --git a/tools/testing/selftests/ve/settings b/tools/testing/selftests/ve/settings
new file mode 100644
index 000000000000..e2206265f67c
--- /dev/null
+++ b/tools/testing/selftests/ve/settings
@@ -0,0 +1 @@
+timeout=900
diff --git a/tools/testing/selftests/ve/ve_vtty_stress.c b/tools/testing/selftests/ve/ve_vtty_stress.c
new file mode 100644
index 000000000000..57edbdeef0c8
--- /dev/null
+++ b/tools/testing/selftests/ve/ve_vtty_stress.c
@@ -0,0 +1,250 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ve_vtty_stress - close a vtty pair from both ends at once.
+ *
+ * A vtty pair is created by the first master open and torn down by the
+ * last close. A close racing the peer's close used to let both sides
+ * decide they were the final one and call release_tty() twice, and a
+ * master opened again while a teardown was still in flight used to hand
+ * out a reference to a tty_struct that was about to be freed. Either
+ * way the next close walked freed memory.
+ *
+ * The master comes from the vzctl ioctl, which does not look the VE up.
+ * The slave is only reachable from inside the VE, as TTY_MAJOR minor
+ * 1..MAX_NR_VTTY_CONSOLES, which vtty_driver() maps to vttys index
+ * minor - 1. So the test makes a VE with cgroup2, hammers the master
+ * from the host and the slave from a task inside it, and fails if the
+ * kernel logs a complaint about the teardown.
+ *
+ * Surviving is not a pass on its own: the first symptom is a warning,
+ * which stops nothing by itself.
+ *
+ * Usage: ve_vtty_stress [seconds] (default 600, or $VE_VTTY_SECS)
+ */
+#define _GNU_SOURCE
+#include <sched.h>
+#include <stdbool.h>
+#include <sys/ioctl.h>
+#include <linux/major.h>
+#include <sys/sysmacros.h>
+#include <sys/wait.h>
+#include <time.h>
+
+#include "ve_selftest.h"
+#include "../kselftest.h"
+
+/* The ioctl argument; the trailing data[] the uapi struct has is unused. */
+struct vzctl_ve_configure {
+	unsigned int veid, key, val, size;
+};
+
+#define VZCTL_VE_CONFIGURE _IOW('.', 15, struct vzctl_ve_configure)
+#define VE_CONFIGURE_OPEN_TTY 5
+
+#define VTTY_IDX 0 /* the pair both ends work on */
+#define SLAVE_MINOR (VTTY_IDX + 1) /* vtty_driver(): index = minor - 1 */
+#define DEFAULT_SECS 600
+
+static int open_master(int ctl_fd, unsigned int veid)
+{
+	struct vzctl_ve_configure c = {
+		.veid = veid,
+		.key = VE_CONFIGURE_OPEN_TTY,
+		.val = VTTY_IDX,
+	};
+
+	return ioctl(ctl_fd, VZCTL_VE_CONFIGURE, &c);
+}
+
+/*
+ * The slave end, inside the VE. Every error here is one the race is
+ * expected to produce: the pair is gone once the master has closed it.
+ */
+static int hammer_slave(time_t deadline, unsigned long *opened)
+{
+	char node[] = "/tmp/ve_vtty_slave";
+
+	unlink(node);
+	if (mknod(node, S_IFCHR | 0600, makedev(TTY_MAJOR, SLAVE_MINOR)) < 0)
+		return -1;
+
+	while (time(NULL) < deadline) {
+		int fd = open(node, O_RDWR | O_NOCTTY | O_NONBLOCK);
+
+		if (fd >= 0) {
+			(*opened)++;
+			close(fd);
+		}
+	}
+	unlink(node);
+	return 0;
+}
+
+/* The kernel log, since a warning fails nothing by itself. */
+static int kmsg_open(void)
+{
+	int fd = open("/dev/kmsg", O_RDONLY | O_NONBLOCK);
+
+	if (fd >= 0)
+		lseek(fd, 0, SEEK_END);
+	return fd;
+}
+
+static int kmsg_check(int fd)
+{
+	static const char *const ours[] = {
+		"tty_release",	    "release_tty",   "check_tty_count",
+		"vtty_open_master", "vtty_shutdown",
+	};
+	char buf[8192];
+	int found = 0;
+	ssize_t r;
+	size_t i;
+
+	while (fd >= 0 && (r = read(fd, buf, sizeof(buf) - 1)) > 0) {
+		buf[r] = '\0';
+		if (!strstr(buf, "WARNING") && !strstr(buf, "BUG") &&
+		    !strstr(buf, "Oops") && !strstr(buf, "general protection"))
+			continue;
+		for (i = 0; i < ARRAY_SIZE(ours); i++)
+			if (strstr(buf, ours[i])) {
+				ksft_print_msg("kernel log: %s", buf);
+				found++;
+				break;
+			}
+	}
+	return found;
+}
+
+int main(int argc, char **argv)
+{
+	unsigned long master_opened = 0, slave_opened = 0;
+	int cg, ctid, ctl, kmsg, ready[2], count[2], status;
+	const char *env = getenv("VE_VTTY_SECS");
+	char path[PATH_MAX], val[16], c;
+	time_t deadline;
+	long secs;
+	pid_t pid;
+
+	ksft_print_header();
+	secs = argc > 1 ? atol(argv[1]) : (env ? atol(env) : DEFAULT_SECS);
+	if (secs <= 0)
+		secs = DEFAULT_SECS;
+
+	ctl = open("/dev/vzctl", O_RDWR);
+	if (ctl < 0)
+		ksft_exit_skip("cannot open /dev/vzctl: %s, the vz modules "
+			       "(vzdev, vzmon) are needed\n",
+			       strerror(errno));
+
+	cg = mount_cg2_fd();
+	if (cg < 0)
+		ksft_exit_skip("cannot mount cgroup2: %s\n", strerror(errno));
+	write_file_at(cg, "cgroup.subtree_control", VE_CONTROLLERS);
+
+	ctid = make_ve(cg, CTID_MIN);
+	if (ctid < 0)
+		ksft_exit_skip("cannot make a VE cgroup\n");
+
+	/*
+	 * The pair is filed under the veid of the VE, so the two ends only
+	 * meet if the cgroup carries the one the master is opened for.
+	 */
+	snprintf(path, sizeof(path), "%d/ve.veid", ctid);
+	snprintf(val, sizeof(val), "%d", ctid);
+	if (write_file_at(cg, path, val) < 0) {
+		destroy_ve(cg, ctid);
+		ksft_exit_skip("cannot set ve.veid: %s\n", strerror(errno));
+	}
+
+	/* Do not report a failure where the vtty pair cannot be had at all. */
+	status = open_master(ctl, ctid);
+	if (status < 0 && errno != EBUSY) {
+		destroy_ve(cg, ctid);
+		ksft_exit_skip("no vtty master: %s\n", strerror(errno));
+	}
+	if (status >= 0)
+		close(status);
+
+	ksft_set_plan(1);
+	kmsg = kmsg_open();
+	if (kmsg < 0)
+		ksft_print_msg("cannot read /dev/kmsg: %s, only a crash will "
+			       "be noticed\n",
+			       strerror(errno));
+
+	if (pipe(ready) || pipe(count))
+		ksft_exit_fail_msg("pipe: %s\n", strerror(errno));
+
+	deadline = time(NULL) + secs;
+	ksft_print_msg("VE %d, pair %d, %ld seconds\n", ctid, VTTY_IDX, secs);
+
+	pid = fork();
+	if (pid == 0) {
+		close(ready[0]);
+		close(count[0]);
+		if (enter_cgroup(cg, ctid) < 0)
+			_exit(1);
+		/*
+		 * CLONE_NEWVE puts the task in the VE the cgroup names, so
+		 * the slave lookup finds the pair. The pid namespace is
+		 * what a VE wants of its first task.
+		 */
+		if (unshare(CLONE_NEWVE | CLONE_NEWPID | CLONE_NEWNS) < 0)
+			_exit(1);
+		if (fork() == 0) {
+			unsigned long n = 0;
+
+			c = 'r';
+			if (write(ready[1], &c, 1) != 1)
+				_exit(1);
+			if (hammer_slave(deadline, &n) < 0)
+				_exit(1);
+			if (write(count[1], &n, sizeof(n)) != sizeof(n))
+				_exit(1);
+			_exit(0);
+		}
+		wait(&status);
+		_exit(WIFEXITED(status) ? WEXITSTATUS(status) : 1);
+	} else if (pid < 0) {
+		ksft_exit_fail_msg("fork: %s\n", strerror(errno));
+	}
+	close(ready[1]);
+	close(count[1]);
+
+	if (read(ready[0], &c, 1) != 1) {
+		waitpid(pid, &status, 0);
+		destroy_ve(cg, ctid);
+		ksft_exit_fail_msg("the VE side never came up\n");
+	}
+
+	while (time(NULL) < deadline) {
+		int fd = open_master(ctl, ctid);
+
+		if (fd >= 0) {
+			master_opened++;
+			close(fd);
+		}
+	}
+
+	if (read(count[0], &slave_opened, sizeof(slave_opened)) !=
+	    sizeof(slave_opened))
+		slave_opened = 0;
+	waitpid(pid, &status, 0);
+
+	ksft_print_msg("%lu master opens, %lu slave opens\n", master_opened,
+		       slave_opened);
+
+	status = kmsg_check(kmsg);
+	destroy_ve(cg, ctid);
+
+	if (!master_opened)
+		ksft_exit_fail_msg("no master was ever opened\n");
+	if (!slave_opened)
+		ksft_exit_fail_msg("no slave was ever opened\n");
+	if (status)
+		ksft_exit_fail_msg("kernel complained %d time(s)\n", status);
+
+	ksft_test_result_pass("vtty pair closed from both ends\n");
+	ksft_finished();
+}
-- 
2.55.0

_______________________________________________
Devel mailing list
Devel@openvz.org
https://lists.openvz.org/mailman/listinfo/devel