I'm running proxmox 6.5.11-7-pve and trying to set up sriov on second port. But it fails.
With unloaded mlx5_core
and now the same for second port it fails:
What am I doing wrong here?
With unloaded mlx5_core
Code:
#lspci|grep -i Mel
0b:00.0 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4]
0b:00.1 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4]
Code:
#modprobe mlx5_core
### dmesg output ###
[Tue Jan 23 09:59:45 2024] mlx5_core 0000:0b:00.0: firmware version: 12.20.1010
[Tue Jan 23 09:59:45 2024] mlx5_core 0000:0b:00.0: 63.008 Gb/s available PCIe bandwidth, limited by 8.0 GT/s PCIe x8 link at 0000:00:03.2 (capable of 126.016 Gb/s with 8.0 GT/s PCIe x16 link)
[Tue Jan 23 09:59:45 2024] mlx5_core 0000:0b:00.0: E-Switch: Total vports 10, per vport: max uc(1024) max mc(16384)
[Tue Jan 23 09:59:45 2024] mlx5_core 0000:0b:00.0: Port module event: module 0, Cable plugged
[Tue Jan 23 09:59:46 2024] mlx5_core 0000:0b:00.0: mlx5e_tc_post_act_init:40:(pid 20550): firmware level support is missing
[Tue Jan 23 09:59:46 2024] mlx5_core 0000:0b:00.0: MLX5E: StrdRq(0) RqSz(1024) StrdSz(256) RxCqeCmprss(0 basic)
[Tue Jan 23 09:59:46 2024] mlx5_core 0000:0b:00.1: firmware version: 12.20.1010
[Tue Jan 23 09:59:46 2024] mlx5_core 0000:0b:00.1: 63.008 Gb/s available PCIe bandwidth, limited by 8.0 GT/s PCIe x8 link at 0000:00:03.2 (capable of 126.016 Gb/s with 8.0 GT/s PCIe x16 link)
[Tue Jan 23 09:59:46 2024] mlx5_core 0000:0b:00.0 enp11s0f0np0: renamed from eth0
[Tue Jan 23 09:59:46 2024] mlx5_core 0000:0b:00.1: E-Switch: Total vports 10, per vport: max uc(1024) max mc(16384)
[Tue Jan 23 09:59:46 2024] mlx5_core 0000:0b:00.1: Port module event: module 1, Cable plugged
[Tue Jan 23 09:59:46 2024] mlx5_core 0000:0b:00.1: mlx5e_tc_post_act_init:40:(pid 20550): firmware level support is missing
[Tue Jan 23 09:59:46 2024] mlx5_core 0000:0b:00.1: MLX5E: StrdRq(0) RqSz(1024) StrdSz(256) RxCqeCmprss(0 basic)
[Tue Jan 23 09:59:46 2024] mlx5_core 0000:0b:00.1 enp11s0f1np1: renamed from eth0
Code:
#echo 0 > /sys/bus/pci/devices/0000\:0b\:00.0/sriov_drivers_autoprobe
#echo 2 > /sys/bus/pci/devices/0000\:0b\:00.0/sriov_numvfs
### dmesg output ###
[Tue Jan 23 10:02:21 2024] mlx5_core 0000:0b:00.0: E-Switch: Enable: mode(LEGACY), nvfs(2), necvfs(0), active vports(3)
[Tue Jan 23 10:02:21 2024] pci 0000:0b:00.2: [15b3:1014] type 00 class 0x020000
[Tue Jan 23 10:02:21 2024] pci 0000:0b:00.2: enabling Extended Tags
[Tue Jan 23 10:02:21 2024] pci 0000:0b:00.2: Adding to iommu group 34
[Tue Jan 23 10:02:21 2024] pci 0000:0b:00.3: [15b3:1014] type 00 class 0x020000
[Tue Jan 23 10:02:21 2024] pci 0000:0b:00.3: enabling Extended Tags
[Tue Jan 23 10:02:21 2024] pci 0000:0b:00.3: Adding to iommu group 35
Code:
#lspci|grep -i Mel
0b:00.0 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4]
0b:00.1 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4]
0b:00.2 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function]
0b:00.3 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4 Virtual Function]
Code:
#mstconfig -d 0b:00.0 q
Device #1:
----------
Device type: ConnectX4
Name: N/A
Description: N/A
Device: 0b:00.0
Configurations: Next Boot
ROCE_NEXT_PROTOCOL 254
NON_PREFETCHABLE_PF_BAR False(0)
NUM_PF_MSIX_VALID True(1)
NUM_OF_VFS 8
NUM_OF_PF 2
SRIOV_EN True(1)
PF_LOG_BAR_SIZE 5
VF_LOG_BAR_SIZE 1
NUM_PF_MSIX 63
NUM_VF_MSIX 11
INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0)
CQE_COMPRESSION BALANCED(0)
PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)
LRO_LOG_TIMEOUT0 6
LRO_LOG_TIMEOUT1 7
LRO_LOG_TIMEOUT2 8
LRO_LOG_TIMEOUT3 12
LOG_DCR_HASH_TABLE_SIZE 14
DCR_LIFO_SIZE 16384
LINK_TYPE_P1 ETH(2)
LINK_TYPE_P2 ETH(2)
ROCE_CC_PRIO_MASK_P1 0
ROCE_CC_PRIO_MASK_P2 0
CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1)
CLAMP_TGT_RATE_P1 False(0)
RPG_TIME_RESET_P1 300
RPG_BYTE_RESET_P1 32767
RPG_THRESHOLD_P1 5
RPG_MAX_RATE_P1 0
RPG_AI_RATE_P1 5
RPG_HAI_RATE_P1 50
RPG_GD_P1 11
RPG_MIN_DEC_FAC_P1 50
RPG_MIN_RATE_P1 1
RATE_TO_SET_ON_FIRST_CNP_P1 0
DCE_TCP_G_P1 4
DCE_TCP_RTT_P1 1
RATE_REDUCE_MONITOR_PERIOD_P1 4
INITIAL_ALPHA_VALUE_P1 1023
MIN_TIME_BETWEEN_CNPS_P1 0
CNP_802P_PRIO_P1 6
CNP_DSCP_P1 48
CLAMP_TGT_RATE_AFTER_TIME_INC_P2 True(1)
CLAMP_TGT_RATE_P2 False(0)
RPG_TIME_RESET_P2 300
RPG_BYTE_RESET_P2 32767
RPG_THRESHOLD_P2 5
RPG_MAX_RATE_P2 0
RPG_AI_RATE_P2 5
RPG_HAI_RATE_P2 50
RPG_GD_P2 11
RPG_MIN_DEC_FAC_P2 50
RPG_MIN_RATE_P2 1
RATE_TO_SET_ON_FIRST_CNP_P2 0
DCE_TCP_G_P2 4
DCE_TCP_RTT_P2 1
RATE_REDUCE_MONITOR_PERIOD_P2 4
INITIAL_ALPHA_VALUE_P2 1023
MIN_TIME_BETWEEN_CNPS_P2 0
CNP_802P_PRIO_P2 6
CNP_DSCP_P2 48
LLDP_NB_DCBX_P1 False(0)
LLDP_NB_RX_MODE_P1 OFF(0)
LLDP_NB_TX_MODE_P1 OFF(0)
LLDP_NB_DCBX_P2 False(0)
LLDP_NB_RX_MODE_P2 OFF(0)
LLDP_NB_TX_MODE_P2 OFF(0)
DCBX_IEEE_P1 True(1)
DCBX_CEE_P1 True(1)
DCBX_WILLING_P1 True(1)
DCBX_IEEE_P2 True(1)
DCBX_CEE_P2 True(1)
DCBX_WILLING_P2 True(1)
KEEP_ETH_LINK_UP_P1 True(1)
KEEP_IB_LINK_UP_P1 False(0)
KEEP_LINK_UP_ON_BOOT_P1 False(0)
KEEP_LINK_UP_ON_STANDBY_P1 False(0)
KEEP_ETH_LINK_UP_P2 True(1)
KEEP_IB_LINK_UP_P2 False(0)
KEEP_LINK_UP_ON_BOOT_P2 False(0)
KEEP_LINK_UP_ON_STANDBY_P2 False(0)
NUM_OF_VL_P1 _4_VLs(3)
NUM_OF_TC_P1 _8_TCs(0)
NUM_OF_PFC_P1 8
NUM_OF_VL_P2 _4_VLs(3)
NUM_OF_TC_P2 _8_TCs(0)
NUM_OF_PFC_P2 8
DUP_MAC_ACTION_P1 LAST_CFG(0)
DUP_MAC_ACTION_P2 LAST_CFG(0)
PORT_OWNER True(1)
ALLOW_RD_COUNTERS True(1)
RENEG_ON_CHANGE True(1)
TRACER_ENABLE False(0)
IP_VER IPv4(0)
BOOT_UNDI_NETWORK_WAIT 0
BOOT_DBG_LOG False(0)
UEFI_LOGS DISABLED(0)
BOOT_VLAN 1
LEGACY_BOOT_PROTOCOL PXE(1)
BOOT_VLAN_EN False(0)
BOOT_PKEY 0
ADVANCED_PCI_SETTINGS False(0)
#mstconfig -d 0b:00.1 q
Device #1:
----------
Device type: ConnectX4
Name: N/A
Description: N/A
Device: 0b:00.1
Configurations: Next Boot
ROCE_NEXT_PROTOCOL 254
NON_PREFETCHABLE_PF_BAR False(0)
NUM_PF_MSIX_VALID True(1)
NUM_OF_VFS 8
NUM_OF_PF 2
SRIOV_EN True(1)
PF_LOG_BAR_SIZE 5
VF_LOG_BAR_SIZE 1
NUM_PF_MSIX 63
NUM_VF_MSIX 11
INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0)
CQE_COMPRESSION BALANCED(0)
PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)
LRO_LOG_TIMEOUT0 6
LRO_LOG_TIMEOUT1 7
LRO_LOG_TIMEOUT2 8
LRO_LOG_TIMEOUT3 12
LOG_DCR_HASH_TABLE_SIZE 14
DCR_LIFO_SIZE 16384
LINK_TYPE_P1 ETH(2)
LINK_TYPE_P2 ETH(2)
ROCE_CC_PRIO_MASK_P1 0
ROCE_CC_PRIO_MASK_P2 0
CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1)
CLAMP_TGT_RATE_P1 False(0)
RPG_TIME_RESET_P1 300
RPG_BYTE_RESET_P1 32767
RPG_THRESHOLD_P1 5
RPG_MAX_RATE_P1 0
RPG_AI_RATE_P1 5
RPG_HAI_RATE_P1 50
RPG_GD_P1 11
RPG_MIN_DEC_FAC_P1 50
RPG_MIN_RATE_P1 1
RATE_TO_SET_ON_FIRST_CNP_P1 0
DCE_TCP_G_P1 4
DCE_TCP_RTT_P1 1
RATE_REDUCE_MONITOR_PERIOD_P1 4
INITIAL_ALPHA_VALUE_P1 1023
MIN_TIME_BETWEEN_CNPS_P1 0
CNP_802P_PRIO_P1 6
CNP_DSCP_P1 48
CLAMP_TGT_RATE_AFTER_TIME_INC_P2 True(1)
CLAMP_TGT_RATE_P2 False(0)
RPG_TIME_RESET_P2 300
RPG_BYTE_RESET_P2 32767
RPG_THRESHOLD_P2 5
RPG_MAX_RATE_P2 0
RPG_AI_RATE_P2 5
RPG_HAI_RATE_P2 50
RPG_GD_P2 11
RPG_MIN_DEC_FAC_P2 50
RPG_MIN_RATE_P2 1
RATE_TO_SET_ON_FIRST_CNP_P2 0
DCE_TCP_G_P2 4
DCE_TCP_RTT_P2 1
RATE_REDUCE_MONITOR_PERIOD_P2 4
INITIAL_ALPHA_VALUE_P2 1023
MIN_TIME_BETWEEN_CNPS_P2 0
CNP_802P_PRIO_P2 6
CNP_DSCP_P2 48
LLDP_NB_DCBX_P1 False(0)
LLDP_NB_RX_MODE_P1 OFF(0)
LLDP_NB_TX_MODE_P1 OFF(0)
LLDP_NB_DCBX_P2 False(0)
LLDP_NB_RX_MODE_P2 OFF(0)
LLDP_NB_TX_MODE_P2 OFF(0)
DCBX_IEEE_P1 True(1)
DCBX_CEE_P1 True(1)
DCBX_WILLING_P1 True(1)
DCBX_IEEE_P2 True(1)
DCBX_CEE_P2 True(1)
DCBX_WILLING_P2 True(1)
KEEP_ETH_LINK_UP_P1 True(1)
KEEP_IB_LINK_UP_P1 False(0)
KEEP_LINK_UP_ON_BOOT_P1 False(0)
KEEP_LINK_UP_ON_STANDBY_P1 False(0)
KEEP_ETH_LINK_UP_P2 True(1)
KEEP_IB_LINK_UP_P2 False(0)
KEEP_LINK_UP_ON_BOOT_P2 False(0)
KEEP_LINK_UP_ON_STANDBY_P2 False(0)
NUM_OF_VL_P1 _4_VLs(3)
NUM_OF_TC_P1 _8_TCs(0)
NUM_OF_PFC_P1 8
NUM_OF_VL_P2 _4_VLs(3)
NUM_OF_TC_P2 _8_TCs(0)
NUM_OF_PFC_P2 8
DUP_MAC_ACTION_P1 LAST_CFG(0)
DUP_MAC_ACTION_P2 LAST_CFG(0)
PORT_OWNER True(1)
ALLOW_RD_COUNTERS True(1)
RENEG_ON_CHANGE True(1)
TRACER_ENABLE False(0)
IP_VER IPv4(0)
BOOT_UNDI_NETWORK_WAIT 0
BOOT_DBG_LOG False(0)
UEFI_LOGS DISABLED(0)
BOOT_VLAN 1
LEGACY_BOOT_PROTOCOL PXE(1)
BOOT_VLAN_EN False(0)
BOOT_PKEY 0
ADVANCED_PCI_SETTINGS False(0)
Code:
#echo 0 > /sys/bus/pci/devices/0000\:0b\:00.1/sriov_drivers_autoprobe
#echo 2 > /sys/bus/pci/devices/0000\:0b\:00.1/sriov_numvfs
### dmesg output ###
[Tue Jan 23 10:03:24 2024] mlx5_core 0000:0b:00.1: E-Switch: Enable: mode(LEGACY), nvfs(2), necvfs(0), active vports(3)
[Tue Jan 23 10:03:24 2024] pci 0000:0b:01.2: [15b3:1014] type 7f class 0xffffff
[Tue Jan 23 10:03:24 2024] pci 0000:0b:01.2: unknown header type 7f, ignoring device
[Tue Jan 23 10:03:25 2024] mlx5_core 0000:0b:00.1: mlx5_sriov_enable:195:(pid 16825): pci_enable_sriov failed : -5
[Tue Jan 23 10:03:25 2024] mlx5_core 0000:0b:00.1: E-Switch: Unload vfs: mode(LEGACY), nvfs(2), necvfs(0), active vports(3)
[Tue Jan 23 10:03:25 2024] mlx5_core 0000:0b:00.1: E-Switch: Disable: mode(LEGACY), nvfs(2), necvfs(0), active vports(1)
Code:
cat /sys/bus/pci/devices/0000\:0b\:00.0/sriov_numvfs
2
cat /sys/bus/pci/devices/0000\:0b\:00.0/sriov_totalvfs
8
cat /sys/bus/pci/devices/0000\:0b\:00.1/sriov_numvfs
0
cat /sys/bus/pci/devices/0000\:0b\:00.1/sriov_totalvfs
8