[Devel] [PATCH hci-8.1 v5 1/5] selftests/ve: check CPUID override emulation for VE tasks

[Konstantin Khorenko <khorenko@virtuozzo.com>]

Write a one-entry /proc/vz/cpuid_override table that clears one leaf 1
feature bit and compare what a task in a fresh VE and the host see:
the VE task must get the table values, the host must keep the hardware
bit. A second test flushes the table after the task was armed and
expects the hardware bit back, since faulting stays on for the task
and the emulation has to fall back to the real CPUID. On its own that
test would also pass on a kernel that never arms faulting; the first
one proves the arming.

Faulting is switched on for a task at its next context switch, not
when the flag is set, so the child stops itself with SIGSTOP after
entering the VE and the parent resumes it before the CPUID. The stop
is reported to the parent before the child leaves the CPU, and a
SIGCONT in that window would let it run on without a switch, so the
child hands over its voluntary context switch count first and the
parent waits for it to grow before the SIGCONT. For the second test
the table is flushed while the child stands still.

Without commit 07338fbae5a3 ("x86/traps: emulate CPUID faults for user
tasks in exc_general_protection()") both children die with SIGSEGV.
The tests skip on CPUs without CPUID faulting and restore the previous
table on exit; a table the kernel cannot take back in one write fails
the setup instead. The harness may skip FIXTURE_TEARDOWN once an ASSERT
has fired, so the cleanup is a FIXTURE_TEARDOWN_PARENT, which runs
exactly once whether or not one did; it also kills a child that such
an ASSERT left stopped in the VE cgroup. The test is built on x86 only.

https://virtuozzo.atlassian.net/browse/VSTOR-148843

Feature: x86: cpu pools (cpuid override)
Signed-off-by: Konstantin Khorenko <khorenko@virtuozzo.com>
---
 tools/testing/selftests/ve/.gitignore              |   1 +
 tools/testing/selftests/ve/Makefile                |  10 +
 .../testing/selftests/ve/ve_cpuid_override_test.c  | 296 +++++++++++++++++++++
 3 files changed, 307 insertions(+)

diff --git a/tools/testing/selftests/ve/.gitignore b/tools/testing/selftests/ve/.gitignore
index 3df4d05888dce..bdc322c645d95 100644
--- a/tools/testing/selftests/ve/.gitignore
+++ b/tools/testing/selftests/ve/.gitignore
@@ -1,3 +1,4 @@
 ve_ns_owner_test
 ve_perms_test
 ve_mount_accounting_test
+ve_cpuid_override_test
diff --git a/tools/testing/selftests/ve/Makefile b/tools/testing/selftests/ve/Makefile
index c6efe7c4b4fba..d919c1c289c8b 100644
--- a/tools/testing/selftests/ve/Makefile
+++ b/tools/testing/selftests/ve/Makefile
@@ -6,4 +6,14 @@ TEST_GEN_PROGS += ve_ns_owner_test
 TEST_GEN_PROGS += ve_perms_test
 TEST_GEN_PROGS += ve_mount_accounting_test
 
+# ARCH stays untouched: lib.mk picks the clang target by its emptiness.
+ifeq ($(ARCH),)
+ve_arch := $(shell uname -m 2>/dev/null || echo not)
+else
+ve_arch := $(ARCH)
+endif
+ifeq ($(ve_arch),$(filter $(ve_arch),x86 x86_64))
+TEST_GEN_PROGS += ve_cpuid_override_test
+endif
+
 include ../lib.mk
diff --git a/tools/testing/selftests/ve/ve_cpuid_override_test.c b/tools/testing/selftests/ve/ve_cpuid_override_test.c
new file mode 100644
index 0000000000000..398f1fef5d246
--- /dev/null
+++ b/tools/testing/selftests/ve/ve_cpuid_override_test.c
@@ -0,0 +1,296 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * /proc/vz/cpuid_override selftests
+ *
+ * A non-empty override table turns CPUID faulting on for every task that
+ * enters a VE, and the #GP handler has to emulate the instruction for
+ * them. The tests check that a task in a VE sees the table values, that
+ * the host keeps seeing the hardware values, and that a task which was
+ * armed while the table was set falls back to the hardware once the
+ * table is flushed. A kernel that does not emulate the fault kills the
+ * child with SIGSEGV instead.
+ *
+ * The table is global: run this on a node without live Containers and one
+ * instance at a time, a task entering a VE while a case runs is armed for
+ * life. A harness timeout kills the test without its teardown and leaves
+ * the test table in place and the VE cgroup behind.
+ */
+#define _GNU_SOURCE
+#include <cpuid.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/wait.h>
+
+#include "../kselftest_harness.h"
+#include "ve_selftest.h"
+
+#define CPUID_OVERRIDE_PATH	"/proc/vz/cpuid_override"
+#define CPUID_LEAF		1
+#define CPUID_LINE_LEN		128
+/* The kernel takes a table back only in one write below a page. */
+#define CPUID_WRITE_MAX		4096
+#define CPUID_SAVED_MAX		(CPUID_WRITE_MAX + 1)
+#define SWITCH_WAIT_ROUNDS	100000
+
+struct cpuid_regs {
+	unsigned int eax, ebx, ecx, edx;
+};
+
+static void do_cpuid(unsigned int leaf, struct cpuid_regs *r)
+{
+	__cpuid_count(leaf, 0, r->eax, r->ebx, r->ecx, r->edx);
+}
+
+/* Voluntary context switches of a task, from its /proc status file. */
+static int read_nvcsw(const char *status, unsigned long *nvcsw)
+{
+	char buf[8192], *p;
+
+	if (read_file_at(AT_FDCWD, status, buf, sizeof(buf)) <= 0)
+		return -1;
+	p = strstr(buf, "\nvoluntary_ctxt_switches:");
+	if (!p || sscanf(p, "\nvoluntary_ctxt_switches: %lu", nvcsw) != 1)
+		return -1;
+	return 0;
+}
+
+struct child_ctx {
+	int result_pipe[2];
+};
+
+/*
+ * The child enters a fresh VE, so it is armed with the table state of that
+ * moment, then reports its view of the leaf to the parent over a pipe. It
+ * stops itself in between: a stopped task is off the CPU, so the resume is
+ * a context switch, and the parent may change the table while the child
+ * stands still. It tells the parent its context switch count first, so
+ * that the parent can wait for the switch out.
+ */
+static int cpuid_child(struct child_ctx *ctx)
+{
+	struct cpuid_regs r;
+	unsigned long nvcsw;
+
+	close(ctx->result_pipe[0]);
+
+	if (unshare(CLONE_NEWVE) < 0)
+		return 21;
+	if (read_nvcsw("/proc/self/status", &nvcsw))
+		return 22;
+	if (write(ctx->result_pipe[1], &nvcsw, sizeof(nvcsw)) != sizeof(nvcsw))
+		return 23;
+	if (raise(SIGSTOP))
+		return 24;
+
+	do_cpuid(CPUID_LEAF, &r);
+
+	if (write(ctx->result_pipe[1], &r, sizeof(r)) != sizeof(r))
+		return 25;
+	return 0;
+}
+
+FIXTURE(ve_cpuid_override)
+{
+	int cgv2_fd;
+	int ctid;
+	struct cpuid_regs host;
+	unsigned int bit;
+	pid_t child;
+	char table[CPUID_LINE_LEN];
+	char saved[CPUID_SAVED_MAX];
+};
+
+FIXTURE_SETUP(ve_cpuid_override)
+{
+	int fd, len;
+
+	if (geteuid())
+		SKIP(return, "needs root");
+	fd = open(CPUID_OVERRIDE_PATH, O_RDONLY);
+	if (fd < 0)
+		SKIP(return, "%s: %s", CPUID_OVERRIDE_PATH, strerror(errno));
+	close(fd);
+
+	len = read_file_at(AT_FDCWD, CPUID_OVERRIDE_PATH, self->saved,
+			   sizeof(self->saved));
+	ASSERT_GE(len, 0) TH_LOG("%s: %s", CPUID_OVERRIDE_PATH, strerror(errno));
+	ASSERT_LT(len, CPUID_WRITE_MAX)
+		TH_LOG("a table of at least %d bytes cannot be restored", len);
+
+	/* Mask one feature bit the host has; SSE2 when present. */
+	do_cpuid(CPUID_LEAF, &self->host);
+	self->bit = 1u << 26;
+	if (!(self->host.edx & self->bit))
+		self->bit = self->host.edx & -self->host.edx;
+	ASSERT_NE(self->bit, 0u);
+
+	snprintf(self->table, sizeof(self->table),
+		 "0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n", CPUID_LEAF,
+		 self->host.eax, self->host.ebx, self->host.ecx,
+		 self->host.edx & ~self->bit);
+
+	/* Last, so that a failed ASSERT above leaves no VE cgroup behind. */
+	self->cgv2_fd = mount_cg2_fd();
+	ASSERT_GE(self->cgv2_fd, 0) TH_LOG("cgroup2 mount: %s", strerror(errno));
+	ASSERT_EQ(write_file_at(self->cgv2_fd, "cgroup.subtree_control",
+				VE_CONTROLLERS), 0);
+	self->ctid = make_ve(self->cgv2_fd, CTID_MIN);
+	ASSERT_GE(self->ctid, 0);
+}
+
+/*
+ * Runs after the test body in whichever process the harness picks, the test
+ * process itself after a failed ASSERT, so the table is restored whether or
+ * not one fired. The setup fd may be gone by then, hence the fresh mount.
+ */
+FIXTURE_TEARDOWN_PARENT(ve_cpuid_override)
+{
+	int cgv2_fd;
+
+	EXPECT_EQ(write_file_at(AT_FDCWD, CPUID_OVERRIDE_PATH,
+				self->saved[0] ? self->saved : "\n"), 0)
+		TH_LOG("could not restore %s: %s", CPUID_OVERRIDE_PATH,
+		       strerror(errno));
+
+	/* A child left stopped by a failed ASSERT keeps the VE cgroup busy. */
+	if (self->child > 0)
+		kill(self->child, SIGKILL);
+
+	cgv2_fd = mount_cg2_fd();
+	if (cgv2_fd < 0) {
+		TH_LOG("cgroup2 mount: %s, VE cgroup %d left behind",
+		       strerror(errno), self->ctid);
+		return;
+	}
+	destroy_ve(cgv2_fd, self->ctid);
+	close(cgv2_fd);
+}
+
+/*
+ * Stopped is reported to the parent before the child leaves the CPU, so a
+ * SIGCONT sent right away could keep it running without a switch. Wait
+ * until its voluntary context switch count has moved on.
+ */
+static void wait_switched_out(struct __test_metadata *_metadata, pid_t pid,
+			      unsigned long nvcsw_before)
+{
+	char status[64];
+	unsigned long nvcsw;
+	int i;
+
+	snprintf(status, sizeof(status), "/proc/%d/status", pid);
+	for (i = 0; i < SWITCH_WAIT_ROUNDS; i++) {
+		ASSERT_EQ(read_nvcsw(status, &nvcsw), 0);
+		if (nvcsw > nvcsw_before)
+			return;
+		sched_yield();
+	}
+	ASSERT_GT(nvcsw, nvcsw_before);
+}
+
+/* Fork the child into the VE and return once it is off the CPU. */
+static pid_t spawn_cpuid_child(struct __test_metadata *_metadata,
+			       FIXTURE_DATA(ve_cpuid_override) * self,
+			       struct child_ctx *ctx)
+{
+	unsigned long nvcsw;
+	int status;
+	pid_t pid;
+
+	ASSERT_EQ(pipe(ctx->result_pipe), 0);
+	ASSERT_GE(enter_cgroup(self->cgv2_fd, self->ctid), 0);
+
+	pid = fork();
+	if (pid == 0)
+		_exit(cpuid_child(ctx));
+	if (pid > 0)
+		self->child = pid;
+
+	/* Leave the VE cgroup before any ASSERT, or destroy_ve() gets EBUSY. */
+	ASSERT_GE(enter_cgroup(self->cgv2_fd, 0), 0);
+	ASSERT_GE(pid, 0);
+	close(ctx->result_pipe[1]);
+
+	ASSERT_GE(waitpid(pid, &status, WUNTRACED), 0);
+	if (!WIFSTOPPED(status)) {
+		/* Reaped: the pid may be reused, keep it from teardown. */
+		self->child = 0;
+		if (WIFEXITED(status))
+			TH_LOG("child exited with %d", WEXITSTATUS(status));
+		if (WIFSIGNALED(status))
+			TH_LOG("child killed by signal %d", WTERMSIG(status));
+	}
+	ASSERT_TRUE(WIFSTOPPED(status));
+	ASSERT_EQ(read(ctx->result_pipe[0], &nvcsw, sizeof(nvcsw)),
+		  (int)sizeof(nvcsw));
+	wait_switched_out(_metadata, pid, nvcsw);
+	return pid;
+}
+
+static void resume_cpuid_child(struct __test_metadata *_metadata, pid_t pid)
+{
+	ASSERT_EQ(kill(pid, SIGCONT), 0);
+}
+
+static void collect_cpuid_child(struct __test_metadata *_metadata,
+				FIXTURE_DATA(ve_cpuid_override) * self,
+				struct child_ctx *ctx, struct cpuid_regs *r)
+{
+	int status, len;
+
+	len = read(ctx->result_pipe[0], r, sizeof(*r));
+	ASSERT_GE(waitpid(self->child, &status, 0), 0);
+	self->child = 0;
+	if (WIFSIGNALED(status))
+		TH_LOG("child killed by signal %d", WTERMSIG(status));
+	ASSERT_TRUE(WIFEXITED(status));
+	ASSERT_EQ(WEXITSTATUS(status), 0);
+	ASSERT_EQ(len, (int)sizeof(*r));
+
+	close(ctx->result_pipe[0]);
+}
+
+TEST_F(ve_cpuid_override, ve_sees_table_host_sees_hardware)
+{
+	struct child_ctx ctx;
+	struct cpuid_regs r;
+	pid_t pid;
+
+	ASSERT_EQ(write_file_at(AT_FDCWD, CPUID_OVERRIDE_PATH, self->table), 0);
+
+	pid = spawn_cpuid_child(_metadata, self, &ctx);
+	resume_cpuid_child(_metadata, pid);
+	collect_cpuid_child(_metadata, self, &ctx, &r);
+
+	EXPECT_EQ(r.eax, self->host.eax);
+	EXPECT_EQ(r.ebx, self->host.ebx);
+	EXPECT_EQ(r.ecx, self->host.ecx);
+	EXPECT_EQ(r.edx, self->host.edx & ~self->bit);
+
+	do_cpuid(CPUID_LEAF, &r);
+	EXPECT_NE(r.edx & self->bit, 0u);
+}
+
+TEST_F(ve_cpuid_override, flushed_table_falls_back_to_hardware)
+{
+	struct child_ctx ctx;
+	struct cpuid_regs r;
+	pid_t pid;
+
+	ASSERT_EQ(write_file_at(AT_FDCWD, CPUID_OVERRIDE_PATH, self->table), 0);
+
+	pid = spawn_cpuid_child(_metadata, self, &ctx);
+
+	ASSERT_EQ(write_file_at(AT_FDCWD, CPUID_OVERRIDE_PATH, "\n"), 0);
+	resume_cpuid_child(_metadata, pid);
+	collect_cpuid_child(_metadata, self, &ctx, &r);
+
+	EXPECT_NE(r.edx & self->bit, 0u);
+}
+
+TEST_HARNESS_MAIN

-- 
2.43.0

_______________________________________________
Devel mailing list
Devel@openvz.org
https://lists.openvz.org/mailman/listinfo/devel