* [Devel] [PATCH RHEL10 COMMIT] ms/qede: sync udp_tunnel ports outside qede_lock in the recovery path
[not found] <20260731155337.1209007-1-den@openvz.org>
@ 2026-08-05 11:20 ` Konstantin Khorenko
0 siblings, 0 replies; only message in thread
From: Konstantin Khorenko @ 2026-08-05 11:20 UTC (permalink / raw)
The commit is pushed to "branch-rh10-6.12.0-211.39.1.16.x.vz10-ovz" and will appear at git at bitbucket.org:openvz/vzkernel.git
after rh10-6.12.0-211.39.1.16.2.vz10
------>
commit b8673a452aa2f5ad79d424888140d11639ab0384
Author: Denis V. Lunev <den@openvz.org>
Date: Fri Jul 31 17:53:37 2026 +0200
ms/qede: sync udp_tunnel ports outside qede_lock in the recovery path
A TX timeout on a qede NIC that has VXLAN/GENEVE tunnel ports
configured wedges the rtnetlink control plane of the whole machine:
NETDEV WATCHDOG: ens6f1 (qede): transmit queue 2 timed out 10226 ms
[qede_tx_timeout:586(ens6f1)]TX timeout on queue 2!
[qede_recovery_handler:2665(ens6f0)]Starting a recovery process
The recovery path deadlocks on the driver's own mutex:
qede_sp_task
rtnl_lock()
mutex_lock(&edev->qede_lock) <- taken
qede_recovery_handler
qede_load
udp_tunnel_nic_reset_ntf
__udp_tunnel_nic_device_sync
info->sync_table == qede_udp_tunnel_sync
mutex_lock(&edev->qede_lock) <- same task: deadlock
The mutex is not recursive, so the kworker blocks on itself with
rtnl_lock held, and neither lock is ever released. Every task that
calls rtnl_lock() afterwards (ip, ovs-vswitchd, lldpad, IPv6
addrconf, sshd) blocks forever while the node still answers ping.
In a vmcore from an affected production node rtnl_mutex.owner
decodes to the very kworker blocked at the innermost mutex_lock()
above.
Re-sync the tunnel ports from qede_sp_task() after the internal lock
is dropped, still under rtnl_lock as the udp_tunnel API requires.
This mirrors qede_open(), which calls udp_tunnel_nic_reset_ntf()
under rtnl without the internal lock.
qede_recovery_handler() now returns whether it has successfully
reloaded an open device, and the caller re-syncs the ports only in
that case. This keeps the old gating exactly: a device that was down
or a failed recovery returns false, as those paths never reached the
udp_tunnel_nic_reset_ntf() call before either.
This was the only user of the qede_lock()/qede_unlock() helpers, so
remove them.
Fixes: 8cd160a29415 ("qede: convert to new udp_tunnel_nic infra")
Signed-off-by: Denis V. Lunev <den@openvz.org>
CC: Andrew Lunn <andrew+netdev@lunn.ch>
CC: "David S. Miller" <davem@davemloft.net>
CC: Eric Dumazet <edumazet@google.com>
CC: Jakub Kicinski <kuba@kernel.org>
CC: Paolo Abeni <pabeni@redhat.com>
(cherry picked from commit 451c9075d6c53f2438d110addbeeeea6fac18567)
https://virtuozzo.atlassian.net/browse/VSTOR-138358
Feature: fix ms/drivers
---
drivers/net/ethernet/qlogic/qede/qede_main.c | 44 ++++++++++++++--------------
1 file changed, 22 insertions(+), 22 deletions(-)
diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c
index 66ab1b9d65a1a..cdb20606048ae 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_main.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_main.c
@@ -107,7 +107,7 @@ static void qede_remove(struct pci_dev *pdev);
static void qede_shutdown(struct pci_dev *pdev);
static void qede_link_update(void *dev, struct qed_link_output *link);
static void qede_schedule_recovery_handler(void *dev);
-static void qede_recovery_handler(struct qede_dev *edev);
+static bool qede_recovery_handler(struct qede_dev *edev);
static void qede_schedule_hw_err_handler(void *dev,
enum qed_hw_err_type err_type);
static void qede_get_eth_tlv_data(void *edev, void *data);
@@ -1047,21 +1047,6 @@ void __qede_unlock(struct qede_dev *edev)
mutex_unlock(&edev->qede_lock);
}
-/* This version of the lock should be used when acquiring the RTNL lock is also
- * needed in addition to the internal qede lock.
- */
-static void qede_lock(struct qede_dev *edev)
-{
- rtnl_lock();
- __qede_lock(edev);
-}
-
-static void qede_unlock(struct qede_dev *edev)
-{
- __qede_unlock(edev);
- rtnl_unlock();
-}
-
static void qede_periodic_task(struct work_struct *work)
{
struct qede_dev *edev = container_of(work, struct qede_dev,
@@ -1098,6 +1083,8 @@ static void qede_sp_task(struct work_struct *work)
*/
if (test_and_clear_bit(QEDE_SP_RECOVERY, &edev->sp_flags)) {
+ bool reloaded;
+
cancel_delayed_work_sync(&edev->periodic_task);
#ifdef CONFIG_QED_SRIOV
/* SRIOV must be disabled outside the lock to avoid a deadlock.
@@ -1106,9 +1093,17 @@ static void qede_sp_task(struct work_struct *work)
if (pci_num_vf(edev->pdev))
qede_sriov_configure(edev->pdev, 0);
#endif
- qede_lock(edev);
- qede_recovery_handler(edev);
- qede_unlock(edev);
+ rtnl_lock();
+ __qede_lock(edev);
+ reloaded = qede_recovery_handler(edev);
+ __qede_unlock(edev);
+
+ /* The udp_tunnel core synchronously calls back into
+ * qede_udp_tunnel_sync(), which takes the qede lock.
+ */
+ if (reloaded)
+ udp_tunnel_nic_reset_ntf(edev->ndev);
+ rtnl_unlock();
}
__qede_lock(edev);
@@ -2649,9 +2644,13 @@ static void qede_recovery_failed(struct qede_dev *edev)
edev->ops->common->set_power_state(edev->cdev, PCI_D3hot);
}
-static void qede_recovery_handler(struct qede_dev *edev)
+/* Returns true if an open device was successfully reloaded and its
+ * udp_tunnel ports need to be re-synced by the caller.
+ */
+static bool qede_recovery_handler(struct qede_dev *edev)
{
u32 curr_state = edev->state;
+ bool reloaded = false;
int rc;
DP_NOTICE(edev, "Starting a recovery process\n");
@@ -2681,17 +2680,18 @@ static void qede_recovery_handler(struct qede_dev *edev)
goto err;
qede_config_rx_mode(edev->ndev);
- udp_tunnel_nic_reset_ntf(edev->ndev);
+ reloaded = true;
}
edev->state = curr_state;
DP_NOTICE(edev, "Recovery handling is done\n");
- return;
+ return reloaded;
err:
qede_recovery_failed(edev);
+ return false;
}
static void qede_atomic_hw_err_handler(struct qede_dev *edev)
^ permalink raw reply [flat|nested] only message in thread