[Devel] [PATCH hci-8.1 v7 2/2] selftests: ve: introduce vtty pair teardown stress test

[Eva Kurchatova <eva.kurchatova@virtuozzo.com>]

Either end of a vtty pair creates it, and the last close tears it down.

Race condition in older VZKernel vtty implementation allowed both sides
to decide they were the final one and call release_tty() twice, and a
master opening again while a teardown was in flight used to hand out a
reference to a tty_struct about to be freed. Either way, this caused
use-after-free. That issue was fixed in 23abb203a41a ("ve/vtty: fix
use-after-free on concurrent tty close and reopen").

Hammer both ends for ownership correctness & thread safety verification.

The original reproducer needed a few tens of minutes across a dozen
containers, where vzctl interaction with vtty subsystem was a small part
of the overall runtime, therefore it did not fire on every run. This test
opens and closes nothing but the pair, from both ends and without pause,
so the default stays in the tens of seconds and under the timeout of the
suite, and VE_VTTY_SECS or argv[1] allows to control this.

Nothing keeps veids unique and the vtty map is keyed by veid, so a
clash would hand the test a running container's pair. Before writing
ve.veid the test asks the kernel whether a running container holds the
veid and takes the next id when one does, best effort: one started
later is not seen.
It also skips where /tmp is mounted nodev, where mknod() succeeds and
no open does. The log is read every second, and a ring overrun, which
loses records, counts as a complaint.

https://virtuozzo.atlassian.net/browse/VSTOR-144771

Feature: ve selftest
Signed-off-by: Eva Kurchatova <eva.kurchatova@virtuozzo.com>
---
 tools/testing/selftests/ve/.gitignore       |   1 +
 tools/testing/selftests/ve/Makefile         |   3 +-
 tools/testing/selftests/ve/ve_vtty_stress.c | 416 ++++++++++++++++++++
 3 files changed, 419 insertions(+), 1 deletion(-)
 create mode 100644 tools/testing/selftests/ve/ve_vtty_stress.c

diff --git a/tools/testing/selftests/ve/.gitignore b/tools/testing/selftests/ve/.gitignore
index 3df4d05888dc..d8eb5c346c6e 100644
--- a/tools/testing/selftests/ve/.gitignore
+++ b/tools/testing/selftests/ve/.gitignore
@@ -1,3 +1,4 @@
 ve_ns_owner_test
 ve_perms_test
 ve_mount_accounting_test
+ve_vtty_stress
diff --git a/tools/testing/selftests/ve/Makefile b/tools/testing/selftests/ve/Makefile
index c6efe7c4b4fb..d287a73c5e97 100644
--- a/tools/testing/selftests/ve/Makefile
+++ b/tools/testing/selftests/ve/Makefile
@@ -1,9 +1,10 @@
 # SPDX-License-Identifier: GPL-2.0
 # Makefile for ve selftests.
-CFLAGS += -g -Wall -O2
+CFLAGS += -g -Wall -O2 $(KHDR_INCLUDES)
 
 TEST_GEN_PROGS += ve_ns_owner_test
 TEST_GEN_PROGS += ve_perms_test
 TEST_GEN_PROGS += ve_mount_accounting_test
+TEST_GEN_PROGS += ve_vtty_stress
 
 include ../lib.mk
diff --git a/tools/testing/selftests/ve/ve_vtty_stress.c b/tools/testing/selftests/ve/ve_vtty_stress.c
new file mode 100644
index 000000000000..765735ddea45
--- /dev/null
+++ b/tools/testing/selftests/ve/ve_vtty_stress.c
@@ -0,0 +1,416 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ve_vtty_stress - open and close a vtty pair from both ends at once.
+ *
+ * Either end of a vtty pair creates it, and the last close tears it down.
+ *
+ * Hammer both ends. The master comes from the vzctl ioctl, which does
+ * not look the VE up. The slave is reachable only from inside the VE, as
+ * TTY_MAJOR minor 1..MAX_NR_VTTY_CONSOLES, which vtty_driver() maps to
+ * vttys index minor - 1, so the test builds a VE with cgroup2, sets
+ * ve.veid to the cgroup it made, and runs the slave side there.
+ *
+ * A warning does not stop the host, so the run reads the kernel log too.
+ * One read() of /dev/kmsg is one record, so the test matches vtty driver
+ * splats, as well as generic kernel warning/oops/kasan reports.
+ *
+ * Usage: ve_vtty_stress [seconds] (default 30, or $VE_VTTY_SECS)
+ */
+#define _GNU_SOURCE
+#include <sched.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <sys/ioctl.h>
+#include <sys/prctl.h>
+#include <sys/statvfs.h>
+#include <linux/major.h>
+#include <linux/vzcalluser.h>
+#include <sys/sysmacros.h>
+#include <sys/wait.h>
+#include <time.h>
+
+#include "ve_selftest.h"
+#include "../kselftest.h"
+
+#define VTTY_IDX 0 /* The pair both ends work on */
+#define SLAVE_MINOR (VTTY_IDX + 1) /* vtty_driver(): index = minor - 1 */
+#define DEFAULT_SECS 30
+
+/*
+ * Expected open errno codes: the pair may be closing, or gone already,
+ * when an open lands on it.
+ * Anything else is an environment issue or a bug.
+ */
+static bool expected_errno(int err)
+{
+	return err == EBUSY || err == EIO || err == ENXIO;
+}
+
+/* A runner timeout must still reach the cleanup */
+static volatile sig_atomic_t stop;
+
+static void on_signal(int sig)
+{
+	stop = sig;
+}
+
+/* The handler interrupts a wait, the child is still to be reaped */
+static void wait_child(pid_t pid, int *status)
+{
+	*status = 0;
+	while (waitpid(pid, status, 0) < 0 && errno == EINTR)
+		if (stop)
+			kill(pid, SIGKILL);
+}
+
+static int open_master(int ctl_fd, unsigned int veid)
+{
+	struct vzctl_ve_configure c = {
+		.veid = veid,
+		.key = VE_CONFIGURE_OPEN_TTY,
+		.val = VTTY_IDX,
+	};
+
+	return ioctl(ctl_fd, VZCTL_VE_CONFIGURE, &c);
+}
+
+/* Any key but the tty one makes ve_configure() look a running VE up */
+static bool veid_in_use(int ctl_fd, unsigned int veid)
+{
+	struct vzctl_ve_configure c = {
+		.veid = veid,
+		.key = VE_CONFIGURE_OS_RELEASE,
+	};
+
+	return ioctl(ctl_fd, VZCTL_VE_CONFIGURE, &c) < 0 && errno == ENOKEY;
+}
+
+/* The slave end, inside the VE. Returns the errno that stopped it, or 0. */
+static int hammer_slave(const char *node, time_t deadline,
+			unsigned long *opened)
+{
+	int err = 0;
+
+	while (time(NULL) < deadline) {
+		int fd = open(node, O_RDWR | O_NOCTTY | O_NONBLOCK);
+
+		if (fd >= 0) {
+			(*opened)++;
+			close(fd);
+		} else if (!expected_errno(errno)) {
+			err = errno;
+			break;
+		}
+	}
+
+	return err;
+}
+
+/* The kernel log, since a warning fails nothing by itself. */
+static int kmsg_open(void)
+{
+	int fd = open("/dev/kmsg", O_RDONLY | O_NONBLOCK);
+
+	if (fd >= 0)
+		lseek(fd, 0, SEEK_END);
+	return fd;
+}
+
+/*
+ * Check for kernel log warnings, one read() per record, hence
+ * the function name is not checked.
+ */
+static int kmsg_check(int fd)
+{
+	/* How a warning, a splat or a sanitizer report starts in a record */
+	static const char *const reports[] = {
+		"WARNING: ", "Oops", ";BUG", "kernel BUG at", "UBSAN: ",
+		"soft lockup - ", "general protection fault", "INFO: task ",
+		"rcu: INFO: ",
+	};
+	char buf[8192];
+	int found = 0;
+	ssize_t r;
+	size_t i;
+
+	if (fd < 0)
+		return 0;
+
+	for (;;) {
+		r = read(fd, buf, sizeof(buf) - 1);
+		if (r < 0) {
+			/* The ring wrapped since the last read, records are gone */
+			if (errno == EPIPE) {
+				ksft_print_msg("kernel log overrun, records lost\n");
+				found++;
+				continue;
+			}
+			/* Anything but the end of the log is a scan that stopped */
+			if (errno != EAGAIN) {
+				ksft_print_msg("kernel log read: %s\n",
+					       strerror(errno));
+				found++;
+			}
+			break;
+		}
+		if (r == 0)
+			break;
+		buf[r] = '\0';
+		/* Known symptoms of a buggy kernel */
+		if (strstr(buf, "vtty") && (strstr(buf, "tty->count") ||
+					    strstr(buf, "bad slave count"))) {
+			ksft_print_msg("vtty mismatched count log: %s", buf);
+			found++;
+		}
+		for (i = 0; i < ARRAY_SIZE(reports); i++) {
+			if (!strstr(buf, reports[i]))
+				continue;
+			ksft_print_msg("kernel log: %s", buf);
+			found++;
+			break;
+		}
+	}
+	return found;
+}
+
+int main(int argc, char **argv)
+{
+	unsigned long master_opened = 0, slave_opened = 0;
+	int cg, ctid, ctl, kmsg, ready[2], count[2], status;
+	int master_err = 0, slave_err, complaints = 0;
+	const char *env = getenv("VE_VTTY_SECS");
+	char path[PATH_MAX], val[16], c;
+	char dir[] = "/tmp/ve_vtty_XXXXXX";
+	char node[sizeof(dir) + 6];
+	struct namespace_ctx ns;
+	struct sigaction sa;
+	struct statvfs st;
+	time_t deadline, now, last = 0;
+	long secs;
+	pid_t pid, parent;
+
+	ksft_print_header();
+	secs = argc > 1 ? atol(argv[1]) : (env ? atol(env) : DEFAULT_SECS);
+	if (secs <= 0)
+		secs = DEFAULT_SECS;
+
+	ctl = open("/dev/vzctl", O_RDWR);
+	if (ctl < 0)
+		ksft_exit_skip("cannot open /dev/vzctl: %s, the vz modules "
+			       "(vzdev, vzmon) are needed\n",
+			       strerror(errno));
+
+	cg = mount_cg2_fd();
+	if (cg < 0)
+		ksft_exit_skip("cannot mount cgroup2: %s\n", strerror(errno));
+	if (write_file_at(cg, "cgroup.subtree_control", VE_CONTROLLERS) < 0)
+		ksft_exit_skip("cannot enable the ve controller: %s\n",
+			       strerror(errno));
+
+	/* Nothing is made yet, from here on a signal must reach the cleanup */
+	sa.sa_handler = on_signal;
+	sigemptyset(&sa.sa_mask);
+	sa.sa_flags = 0;
+	sigaction(SIGTERM, &sa, NULL);
+	sigaction(SIGINT, &sa, NULL);
+
+	/* The vtty map is keyed by veid, and ve.veid takes a taken one */
+	for (ctid = CTID_MIN; ; ctid++) {
+		ctid = make_ve(cg, ctid);
+		if (ctid < 0)
+			ksft_exit_skip("cannot make a VE cgroup\n");
+		/* Best effort: a running container holds the veid, try the next */
+		if (!veid_in_use(ctl, ctid))
+			break;
+		destroy_ve(cg, ctid);
+	}
+
+	/* The ve.veid must match between master / slave */
+	snprintf(path, sizeof(path), "%d/ve.veid", ctid);
+	snprintf(val, sizeof(val), "%d", ctid);
+	if (write_file_at(cg, path, val) < 0) {
+		status = errno;
+		destroy_ve(cg, ctid);
+		ksft_exit_skip("cannot set ve.veid: %s\n", strerror(status));
+	}
+
+	/* The log is the only detector of a teardown that does not crash */
+	kmsg = kmsg_open();
+	if (kmsg < 0) {
+		status = errno;
+		destroy_ve(cg, ctid);
+		ksft_exit_skip("cannot read /dev/kmsg: %s\n", strerror(status));
+	}
+
+	/* Not a failure where the pair cannot be had, EBUSY says it exists */
+	status = open_master(ctl, ctid);
+	if (status < 0) {
+		status = errno;
+		destroy_ve(cg, ctid);
+		ksft_exit_skip("no vtty master: %s\n", strerror(status));
+	}
+	close(status);
+
+	/*
+	 * Use temp directory for the node the slave end opens.
+	 * Cleaned up by the master side instead of relying on the slave.
+	 */
+	if (!mkdtemp(dir)) {
+		status = errno;
+		destroy_ve(cg, ctid);
+		ksft_exit_skip("mkdtemp: %s\n", strerror(status));
+	}
+	/* A nodev mount takes the mknod() and refuses every open */
+	if (statvfs(dir, &st) == 0 && (st.f_flag & ST_NODEV)) {
+		rmdir(dir);
+		destroy_ve(cg, ctid);
+		ksft_exit_skip("%s is on a nodev filesystem\n", dir);
+	}
+	snprintf(node, sizeof(node), "%s/slave", dir);
+	if (mknod(node, S_IFCHR | 0600, makedev(TTY_MAJOR, SLAVE_MINOR)) < 0) {
+		status = errno;
+		rmdir(dir);
+		destroy_ve(cg, ctid);
+		ksft_exit_skip("mknod: %s\n", strerror(status));
+	}
+
+	ksft_set_plan(1);
+
+	if (pipe(ready) || pipe(count)) {
+		status = errno;
+		unlink(node);
+		rmdir(dir);
+		destroy_ve(cg, ctid);
+		ksft_exit_fail_msg("pipe: %s\n", strerror(status));
+	}
+
+	/* The VE of the cgroup, which the slave side enters */
+	if (prepare_namespaces(&ns, CLONE_NEWVE, cg, ctid) < 0) {
+		unlink(node);
+		rmdir(dir);
+		destroy_ve(cg, ctid);
+		ksft_exit_fail_msg("cannot prepare the VE namespace\n");
+	}
+
+	deadline = time(NULL) + secs;
+	ksft_print_msg("VE %d, pair %d, %ld seconds\n", ctid, VTTY_IDX, secs);
+
+	parent = getpid();
+	pid = fork();
+	if (pid == 0) {
+		unsigned long n = 0;
+		int err;
+
+		/* The parent stops the run, a signal here just ends this side */
+		signal(SIGTERM, SIG_DFL);
+		signal(SIGINT, SIG_DFL);
+		close(ready[0]);
+		close(count[0]);
+		/*
+		 * Hammering on after the run has given up would hold
+		 * the cgroup open against destroy_ve().
+		 */
+		prctl(PR_SET_PDEATHSIG, SIGKILL);
+		if (getppid() != parent)
+			_exit(0);
+		if (enter_cgroup(cg, ctid) < 0 ||
+		    enter_namespaces(&ns, CLONE_NEWVE) < 0)
+			_exit(errno ? errno : EIO);
+
+		c = 'r';
+		if (write(ready[1], &c, 1) != 1)
+			_exit(errno);
+		err = hammer_slave(node, deadline, &n);
+		/* Send the count either way: it says how far it got. */
+		if (write(count[1], &n, sizeof(n)) != sizeof(n))
+			_exit(errno);
+		_exit(err);
+	} else if (pid < 0) {
+		status = errno;
+		close_namespace_fds(&ns);
+		unlink(node);
+		rmdir(dir);
+		destroy_ve(cg, ctid);
+		ksft_exit_fail_msg("fork: %s\n", strerror(status));
+	}
+	close(ready[1]);
+	close(count[1]);
+
+	if (read(ready[0], &c, 1) != 1) {
+		/* A signal lands here too, with the child alive */
+		kill(pid, SIGKILL);
+		wait_child(pid, &status);
+		close_namespace_fds(&ns);
+		unlink(node);
+		rmdir(dir);
+		destroy_ve(cg, ctid);
+		if (stop)
+			ksft_exit_fail_msg("stopped by signal %d\n", (int)stop);
+		ksft_exit_fail_msg("the VE side never came up: %s\n",
+				   strerror(WIFEXITED(status) ?
+					    WEXITSTATUS(status) : EINTR));
+	}
+
+	while (!stop && (now = time(NULL)) < deadline) {
+		int fd = open_master(ctl, ctid);
+
+		if (fd >= 0) {
+			master_opened++;
+			close(fd);
+		} else if (!expected_errno(errno)) {
+			master_err = errno;
+			break;
+		}
+		/* Drain the log as it fills, the ring is small */
+		if (now != last) {
+			complaints += kmsg_check(kmsg);
+			last = now;
+		}
+	}
+
+	if (master_err || stop) {
+		/* The other end runs to its own deadline, stop it */
+		kill(pid, SIGKILL);
+		wait_child(pid, &status);
+		slave_err = 0;
+	} else {
+		if (read(count[0], &slave_opened, sizeof(slave_opened)) !=
+		    sizeof(slave_opened))
+			slave_opened = 0;
+		wait_child(pid, &status);
+		slave_err = WIFEXITED(status) ? WEXITSTATUS(status) : EINTR;
+	}
+
+	ksft_print_msg("%lu master opens, %lu slave opens\n", master_opened,
+		       slave_opened);
+
+	status = complaints + kmsg_check(kmsg);
+	close_namespace_fds(&ns);
+	unlink(node);
+	rmdir(dir);
+	destroy_ve(cg, ctid);
+
+	/* A complaint outranks how the run ended */
+	if (status)
+		ksft_exit_fail_msg("kernel complained %d time(s)\n", status);
+	if (stop)
+		ksft_exit_fail_msg("stopped by signal %d\n", (int)stop);
+	if (master_err)
+		ksft_exit_fail_msg("the master end stopped: %s\n",
+				   strerror(master_err));
+	/* Denied from the first open, by a device policy say */
+	if (!slave_opened && (slave_err == EPERM || slave_err == EACCES ||
+			      slave_err == ENOENT || slave_err == ENODEV))
+		ksft_exit_skip("the slave end cannot be opened: %s\n",
+			       strerror(slave_err));
+	if (slave_err)
+		ksft_exit_fail_msg("the slave end stopped: %s\n",
+				   strerror(slave_err));
+	if (!master_opened)
+		ksft_exit_fail_msg("no master was ever opened\n");
+	if (!slave_opened)
+		ksft_exit_fail_msg("no slave was ever opened\n");
+
+	ksft_test_result_pass("vtty pair closed from both ends\n");
+	ksft_finished();
+}
-- 
2.56.0

_______________________________________________
Devel mailing list
Devel@openvz.org
https://lists.openvz.org/mailman/listinfo/devel