[Devel] [PATCH hci-8.1 v2] KVM: nVMX: Keep the emulated VMX-preemption timer on L1's TSC

[Devel] [PATCH hci-8.1 v2] KVM: nVMX: Keep the emulated VMX-preemption timer on L1's TSC

[Eva Kurchatova]

KVM emulates L2's VMX-preemption timer with an hrtimer, but the hrtimer
runs on CLOCK_MONOTONIC, which NTP slews against the TSC. The VM-exit
then comes late or early in L1's TSC, and vmx_preemption_timer_test
fails.

Arm the hrtimer an eighth early while more than 100us are left, and re-arm
it from its callback until L1's TSC reaches the deadline kept since
commit 850448f35aaf ("KVM: nVMX: Fix VMX preemption timer migration").
Count the timer value left from that deadline too, and wait for a
running callback when moving the timer to another CPU.

Fixes: f4124500c2c1 ("KVM: nVMX: Fully emulate preemption timer")
https://virtuozzo.atlassian.net/browse/VSTOR-152991
Feature: fix ms/kvm

Signed-off-by: Eva Kurchatova <eva.kurchatova@virtuozzo.com>
---

Changes in v2:
- Read L1's TSC once in vmx_calc_preemption_timer_value(), so that the
  value it returns is counted from the same sample as the deadline it
  sets.
- Use hrtimer_cancel() in vmx_migrate_timers(): the callback may now
  re-arm the timer, and hrtimer_try_to_cancel() gives up on a running
  callback, which left the timer on the CPU the vCPU had left.
- Keep vmx_get_preemption_timer_value(), now counted from the deadline,
  so that its caller is unchanged.
- Correct naming, and that the skew only stretches the last 100us.
- Shorter commit message.

Link to v1: http://lore.virtuozzo.com/kernel/010701a1b11e8c72-26d4b079-ff6d-48b4-8769-7d3abff2e701-000000@eu-central-1.amazonses.com/T/#t

 arch/x86/kvm/vmx/nested.c | 73 ++++++++++++++++++++++++++++-----------
 arch/x86/kvm/vmx/vmx.c    |  3 +-
 2 files changed, 54 insertions(+), 22 deletions(-)

diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 8c771d8c35e3..85161d3ee80d 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -2206,15 +2206,59 @@ void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu)
 	vmx->nested.need_vmcs12_to_shadow_sync = false;
 }
 
+/* L1's TSC in ticks of the emulated timer */
+static u64 vmx_preemption_timer_now(struct kvm_vcpu *vcpu)
+{
+	return kvm_read_l1_tsc(vcpu, rdtsc()) >>
+	       VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
+}
+
+/* Ticks left from @now to the deadline, at most the 32-bit timer value */
+static u64 vmx_preemption_timer_left(struct kvm_vcpu *vcpu, u64 now)
+{
+	u64 deadline = to_vmx(vcpu)->nested.preemption_timer_deadline;
+
+	return deadline > now ? min(deadline - now, (u64)U32_MAX) : 0;
+}
+
+/*
+ * The hrtimer runs on CLOCK_MONOTONIC, which adjtimex may skew against the
+ * TSC by up to 10%: fire an eighth early while more than 100us are left, and
+ * let the callback re-arm until the deadline. The skew then only stretches
+ * the last 100us.
+ */
+static ktime_t vmx_preemption_timer_expiry(struct kvm_vcpu *vcpu, u64 ticks)
+{
+	u64 ns = (ticks << VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE) * 1000000;
+
+	do_div(ns, vcpu->arch.virtual_tsc_khz);
+	if (ns > 100 * NSEC_PER_USEC)
+		ns -= ns >> 3;
+	return ktime_add_ns(ktime_get(), ns);
+}
+
+static void vmx_preemption_timer_expire(struct kvm_vcpu *vcpu)
+{
+	to_vmx(vcpu)->nested.preemption_timer_expired = true;
+	kvm_make_request(KVM_REQ_EVENT, vcpu);
+	kvm_vcpu_kick(vcpu);
+}
+
 static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer)
 {
 	struct vcpu_vmx *vmx =
 		container_of(timer, struct vcpu_vmx, nested.preemption_timer);
+	struct kvm_vcpu *vcpu = &vmx->vcpu;
+	u64 now = vmx_preemption_timer_now(vcpu);
+	u64 left = vmx_preemption_timer_left(vcpu, now);
 
-	vmx->nested.preemption_timer_expired = true;
-	kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu);
-	kvm_vcpu_kick(&vmx->vcpu);
+	if (left) {
+		hrtimer_set_expires(timer,
+				    vmx_preemption_timer_expiry(vcpu, left));
+		return HRTIMER_RESTART;
+	}
 
+	vmx_preemption_timer_expire(vcpu);
 	return HRTIMER_NORESTART;
 }
 
@@ -2223,15 +2267,14 @@ static u64 vmx_calc_preemption_timer_value(struct kvm_vcpu *vcpu)
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
 
-	u64 l1_scaled_tsc = kvm_read_l1_tsc(vcpu, rdtsc()) >>
-			    VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
+	u64 l1_scaled_tsc = vmx_preemption_timer_now(vcpu);
 
 	if (!vmx->nested.has_preemption_timer_deadline) {
 		vmx->nested.preemption_timer_deadline =
 			vmcs12->vmx_preemption_timer_value + l1_scaled_tsc;
 		vmx->nested.has_preemption_timer_deadline = true;
 	}
-	return vmx->nested.preemption_timer_deadline - l1_scaled_tsc;
+	return vmx_preemption_timer_left(vcpu, l1_scaled_tsc);
 }
 
 static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu,
@@ -2244,18 +2287,15 @@ static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu,
 	 * a VMExit prior to executing any instructions in the guest.
 	 */
 	if (preemption_timeout == 0) {
-		vmx_preemption_timer_fn(&vmx->nested.preemption_timer);
+		vmx_preemption_timer_expire(vcpu);
 		return;
 	}
 
 	if (vcpu->arch.virtual_tsc_khz == 0)
 		return;
 
-	preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
-	preemption_timeout *= 1000000;
-	do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz);
 	hrtimer_start(&vmx->nested.preemption_timer,
-		      ktime_add_ns(ktime_get(), preemption_timeout),
+		      vmx_preemption_timer_expiry(vcpu, preemption_timeout),
 		      HRTIMER_MODE_ABS_PINNED);
 }
 
@@ -4430,16 +4470,7 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu)
 
 static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu)
 {
-	ktime_t remaining =
-		hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer);
-	u64 value;
-
-	if (ktime_to_ns(remaining) <= 0)
-		return 0;
-
-	value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz;
-	do_div(value, 1000000);
-	return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
+	return vmx_preemption_timer_left(vcpu, vmx_preemption_timer_now(vcpu));
 }
 
 static bool is_vmcs12_ext_field(unsigned long field)
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 00ec97314037..ca554e79ae53 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -8182,7 +8182,8 @@ void vmx_migrate_timers(struct kvm_vcpu *vcpu)
 	if (is_guest_mode(vcpu)) {
 		struct hrtimer *timer = &to_vmx(vcpu)->nested.preemption_timer;
 
-		if (hrtimer_try_to_cancel(timer) == 1)
+		/* Wait for a running callback, it may re-arm the timer */
+		if (hrtimer_cancel(timer))
 			hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
 	}
 }
-- 
2.56.0

_______________________________________________
Devel mailing list
Devel@openvz.org
https://lists.openvz.org/mailman/listinfo/devel

Re: [Devel] [PATCH hci-8.1 v2] KVM: nVMX: Keep the emulated VMX-preemption timer on L1's TSC

[no-reply]

> KVM emulates L2's VMX-preemption timer with an hrtimer, but the hrtimer
> runs on CLOCK_MONOTONIC, which NTP slews against the TSC. The VM-exit
> then comes late or early in L1's TSC, and vmx_preemption_timer_test
> fails.
> 
> Arm the hrtimer an eighth early while more than 100us are left, and re-arm
> it from its callback until L1's TSC reaches the deadline kept since
> commit 850448f35aaf ("KVM: nVMX: Fix VMX preemption timer migration").
> Count the timer value left from that deadline too, and wait for a
> running callback when moving the timer to another CPU.
> 
> Fixes: f4124500c2c1 ("KVM: nVMX: Fully emulate preemption timer")
> https://virtuozzo.atlassian.net/browse/VSTOR-152991
> Feature: fix ms/kvm
> 
> Signed-off-by: Eva Kurchatova <eva.kurchatova@virtuozzo.com>

Sashiko has reviewed this patch and found no issues. It looks great!

-- 
Sashiko AI review · https://sashiko.vzint.dev/#/patchset/20261106151535.138563-1-eva.kurchatova@virtuozzo.com?part=1


_______________________________________________
Devel mailing list
Devel@openvz.org
https://lists.openvz.org/mailman/listinfo/devel

Re: [Devel] [PATCH hci-8.1 v2] KVM: nVMX: Keep the emulated VMX-preemption timer on L1's TSC

[Denis V. Lunev]

On 06.11.2026 17:09, Eva Kurchatova wrote:
> KVM emulates L2's VMX-preemption timer with an hrtimer, but the hrtimer
> runs on CLOCK_MONOTONIC, which NTP slews against the TSC. The VM-exit
> then comes late or early in L1's TSC, and vmx_preemption_timer_test
> fails.

NAK. NTP did not fail the test. The node's TSC did. The journal of
sqvm2-4019 for the boot that produced VSTOR-152991:

  13:58:47 clocksource: timekeeping watchdog on CPU0: Marking clocksource 'tsc' as unstable because the skew is too large:
  13:58:47 clocksource:   Clocksource 'tsc' skewed -42289835 ns (-42 ms) over watchdog 'kvm-clock' interval of 495964472 ns (495 ms)
  13:58:47 tsc: Marking TSC unstable due to clocksource watchdog
  15:38:36 kvm: SMP vm created on host with unstable TSC; guest TSC will not be reliable
  15:39:14 vmx_preemption_timer_test: not ok

sqvm2-4019 is itself a VM, and its hypervisor moved the TSC by 42 ms
within half a second. KVM warned that the guest TSC would not be
reliable 38 seconds before the test ran. With an unstable host TSC,
KVM rewrites L1's TSC offset on every vcpu_load and catches it up
later, so no emulation keyed on L1's TSC can be exact there, including
this patch's.

The test does not need an exact timer either. L2 computes its deadline
after the VMCALL round trip, so the exit may land anywhere in a window
of about 0.8 ms at 2.8 GHz. That is about 700 ppm of disagreement
between CLOCK_MONOTONIC and the TSC over the whole timer, and far more
over the part re-armed after the save/restore. NTP slews at most
500 ppm, so it cannot fail this test on a sane node.

On the other side of the scale, the patch changes timer emulation on
production hosts:

- the hrtimer now fires up to 8 times per period instead of once, and
  each extra firing is an exit from the running L2;
- the callback now follows every change of l1_tsc_offset while L2
  runs. An L2 write to IA32_TSC that L1 does not intercept, S3 with a
  TSC reset, or the unstable-TSC offset rewrite each make it re-arm
  for up to U32_MAX ticks (about 48 s), where the old countdown fired
  on time.

Upstream knows the emulation is imprecise and has not taken a fix
either (https://bugzilla.kernel.org/show_bug.cgi?id=216177, and
https://bugzilla.kernel.org/show_bug.cgi?id=218025 with the same
"L2 PT expiry TSC > L2 TSC deadline" failure). I do not want to carry a
downstream-only rework of it to cover a test run on a broken clock.

What I would like instead:

1. QA: skip vmx_preemption_timer_test on nodes with an unstable TSC.
   The watchdog clears CLOCK_SOURCE_VALID_FOR_HRES on the TSC, which
   drops it from available_clocksource, so the check survives a
   wrapped dmesg:

     grep -qw tsc /sys/devices/system/clocksource/clocksource0/available_clocksource

   sys_clocksource_is_based_on_tsc() from the selftest library is not
   the right guard: it rejects kvm-clock and would skip the test on
   every sqvm.

2. There is one real bug you found along the way, and I would take a
   patch for it on its own. vmx_calc_preemption_timer_value() returns
   preemption_timer_deadline - l1_scaled_tsc unclamped. When the
   deadline has already passed, after RSM from a long SMM handler or
   KVM_SET_NESTED_STATE after a long downtime, it underflows and
   vmx_start_preemption_timer() arms the hrtimer for up to about two
   hours instead of exiting at once. Clamping it to 0 is a few lines
   that do not touch the hrtimer callback. Mainline has the same code,
   so please send it upstream as well.

So there is no v3 of this patch.

Please send the clamp as a new patch to mainstream first.

Den
_______________________________________________
Devel mailing list
Devel@openvz.org
https://lists.openvz.org/mailman/listinfo/devel