From 088438fe6643dce4195e52452fe995c2608b0a48 Mon Sep 17 00:00:00 2001 From: "Suren A. Chilingaryan" Date: Wed, 15 Aug 2018 18:59:14 +0200 Subject: p2p setup --- config.sh | 6 +-- functions.sh | 54 +++++++++++++++++++++ ...l-2_42_5000-MCX354A-QCB_Ax-FlexBoot-3.4.752.bin | Bin 0 -> 796392 bytes ip | 1 + ip-p2p/ipecamera0.sh | 6 +++ ip-p2p/ipecamera3.sh | 11 +++++ ip-p2p/ipecamera4.sh | 11 +++++ ip-p2p/ipecamera5.sh | 9 ++++ ip-p2p/ipecamera6.sh | 8 +++ ip-switch/ipecamera0.sh | 7 +++ ip-switch/ipecamera3.sh | 11 +++++ ip-switch/ipecamera4.sh | 11 +++++ ip-switch/ipecamera5.sh | 9 ++++ ip-switch/ipecamera6.sh | 8 +++ ip/functions.sh | 51 ------------------- ip/ipecamera0.sh | 7 --- ip/ipecamera3.sh | 11 ----- ip/ipecamera4.sh | 11 ----- ip/ipecamera5.sh | 9 ---- ip/ipecamera6.sh | 8 --- update.sh | 8 +-- 21 files changed, 154 insertions(+), 103 deletions(-) create mode 100644 functions.sh create mode 100644 fw/fw-ConnectX3-rel-2_42_5000-MCX354A-QCB_Ax-FlexBoot-3.4.752.bin create mode 120000 ip create mode 100755 ip-p2p/ipecamera0.sh create mode 100755 ip-p2p/ipecamera3.sh create mode 100755 ip-p2p/ipecamera4.sh create mode 100755 ip-p2p/ipecamera5.sh create mode 100755 ip-p2p/ipecamera6.sh create mode 100755 ip-switch/ipecamera0.sh create mode 100755 ip-switch/ipecamera3.sh create mode 100755 ip-switch/ipecamera4.sh create mode 100755 ip-switch/ipecamera5.sh create mode 100755 ip-switch/ipecamera6.sh delete mode 100644 ip/functions.sh delete mode 100755 ip/ipecamera0.sh delete mode 100755 ip/ipecamera3.sh delete mode 100755 ip/ipecamera4.sh delete mode 100755 ip/ipecamera5.sh delete mode 100755 ip/ipecamera6.sh diff --git a/config.sh b/config.sh index 00f7b41..d8d025d 100644 --- a/config.sh +++ b/config.sh @@ -3,9 +3,9 @@ #rmmod mlx4_en mlx4_ib mlx4_core #modprobe mlx4_en -ip link set ens11 mtu 9000 -echo 1000000000 > /proc/sys/kernel/shmmax # 18446744073692774399 -echo 8000 > /proc/sys/vm/nr_hugepages # 0 +#ip link set ens11 mtu 9000 +#echo 1000000000 > /proc/sys/kernel/shmmax # 18446744073692774399 +#echo 8000 > /proc/sys/vm/nr_hugepages # 0 #Checking information diff --git a/functions.sh b/functions.sh new file mode 100644 index 0000000..32bd30e --- /dev/null +++ b/functions.sh @@ -0,0 +1,54 @@ +function mlxreload { + rmmod mlx4_ib &> /dev/null + rmmod mlx4_en &> /dev/null + rmmod mlx4_core + rmmod ib_ucm &> /dev/null + rmmod rdma_ucm &> /dev/null + rmmod ib_uverbs + modprobe ib_uverbs disable_raw_qp_enforcement=1 +# modprobe ib_ucm + modprobe rdma_ucm + modprobe mlx4_core fast_drop=1 log_num_mgm_entry_size=-1 + modprobe mlx4_en +} + +function mlxload { + reload=0 + + param=$(cat /sys/module/mlx4_core/parameters/log_num_mgm_entry_size) + [ $param -ne -1 ] && reload=1 + + [ $reload -eq 1 ] && mlxreload +} + +function portset { + pci=$1 + mode=$2 + ./connectx_port_config -d "$pci" -c "$mode,$mode" &> /dev/null +} + +function mlxconf { + mode=$1 + + mlxload + + pci=$(./connectx_port_config -s | grep PCI | grep -Po "[\da-f]{4}:[\da-f]{2}:[\da-f]{2}\.[\da-f]") + for dev in $pci; do + echo "Configuring: $dev" + portset $dev $mode + done +} + +function ipset { + int="$1" + ip="$2" + + if [ -n "$ip" ]; then + ip link set "$int" up + ip addr flush "$int" + ip addr show dev "$int" | grep "$ip" || ip addr add "$ip/24" dev "$int" + else + ip addr flush "$int" + ip link set "$int" down + fi +} diff --git a/fw/fw-ConnectX3-rel-2_42_5000-MCX354A-QCB_Ax-FlexBoot-3.4.752.bin b/fw/fw-ConnectX3-rel-2_42_5000-MCX354A-QCB_Ax-FlexBoot-3.4.752.bin new file mode 100644 index 0000000..7ee4b40 Binary files /dev/null and b/fw/fw-ConnectX3-rel-2_42_5000-MCX354A-QCB_Ax-FlexBoot-3.4.752.bin differ diff --git a/ip b/ip new file mode 120000 index 0000000..2833425 --- /dev/null +++ b/ip @@ -0,0 +1 @@ +ip-p2p \ No newline at end of file diff --git a/ip-p2p/ipecamera0.sh b/ip-p2p/ipecamera0.sh new file mode 100755 index 0000000..fc91b5d --- /dev/null +++ b/ip-p2p/ipecamera0.sh @@ -0,0 +1,6 @@ +. functions.sh + +mlxconf eth + +ipset enp4s0 192.168.3.80 +ipset enp8s0 192.168.6.80 diff --git a/ip-p2p/ipecamera3.sh b/ip-p2p/ipecamera3.sh new file mode 100755 index 0000000..7a6baaa --- /dev/null +++ b/ip-p2p/ipecamera3.sh @@ -0,0 +1,11 @@ +. functions.sh + +mlxconf eth + +ipset ens1 192.168.36.83 +ipset ens1d1 192.168.3.83 +ipset ens11 192.168.34.83 +ipset ens11d1 192.168.35.83 + +ipset enp6s0f0 #192.168.2.93 +ipset enp7s0f0 #192.168.2.103 diff --git a/ip-p2p/ipecamera4.sh b/ip-p2p/ipecamera4.sh new file mode 100755 index 0000000..1df99fb --- /dev/null +++ b/ip-p2p/ipecamera4.sh @@ -0,0 +1,11 @@ +. functions.sh + +mlxconf eth + +ipset ens1 192.168.34.84 +ipset ens1d1 192.168.2.84 +ipset ens11 192.168.46.84 +ipset ens11d1 192.168.45.84 +ipset enp6s0f0 #192.168.2.94 +ipset enp7s0f0 #192.168.2.104 + diff --git a/ip-p2p/ipecamera5.sh b/ip-p2p/ipecamera5.sh new file mode 100755 index 0000000..1025c3e --- /dev/null +++ b/ip-p2p/ipecamera5.sh @@ -0,0 +1,9 @@ +. functions.sh + +mlxconf eth + +ipset enp6s0 # 192.168.2.85 +ipset enp6s0d1 +ipset enp9s0 192.168.35.85 +ipset enp10s0 192.168.56.85 +ipset enp10s0d1 #192.168.35.85 diff --git a/ip-p2p/ipecamera6.sh b/ip-p2p/ipecamera6.sh new file mode 100755 index 0000000..4dfd521 --- /dev/null +++ b/ip-p2p/ipecamera6.sh @@ -0,0 +1,8 @@ +. functions.sh + +mlxconf eth + +ipset enp5s0 192.168.36.86 +ipset enp5s0d1 192.168.6.86 +ipset enp10s0 192.168.46.86 +ipset enp10s0d1 192.168.56.86 diff --git a/ip-switch/ipecamera0.sh b/ip-switch/ipecamera0.sh new file mode 100755 index 0000000..d714f93 --- /dev/null +++ b/ip-switch/ipecamera0.sh @@ -0,0 +1,7 @@ +. functions.sh + +mlxconf eth + +ipset enp4s0 192.168.2.80 +ipset enp8s0 #192.168.2.90 +ipset enp9s0 #192.168.2.100 diff --git a/ip-switch/ipecamera3.sh b/ip-switch/ipecamera3.sh new file mode 100755 index 0000000..556e47f --- /dev/null +++ b/ip-switch/ipecamera3.sh @@ -0,0 +1,11 @@ +. functions.sh + +mlxconf eth + +ipset ens1 192.168.36.83 +ipset ens1d1 192.168.2.83 +ipset ens11 192.168.34.83 +ipset ens11d1 192.168.35.83 + +ipset enp6s0f0 #192.168.2.93 +ipset enp7s0f0 #192.168.2.103 diff --git a/ip-switch/ipecamera4.sh b/ip-switch/ipecamera4.sh new file mode 100755 index 0000000..1df99fb --- /dev/null +++ b/ip-switch/ipecamera4.sh @@ -0,0 +1,11 @@ +. functions.sh + +mlxconf eth + +ipset ens1 192.168.34.84 +ipset ens1d1 192.168.2.84 +ipset ens11 192.168.46.84 +ipset ens11d1 192.168.45.84 +ipset enp6s0f0 #192.168.2.94 +ipset enp7s0f0 #192.168.2.104 + diff --git a/ip-switch/ipecamera5.sh b/ip-switch/ipecamera5.sh new file mode 100755 index 0000000..8666ae5 --- /dev/null +++ b/ip-switch/ipecamera5.sh @@ -0,0 +1,9 @@ +. functions.sh + +mlxconf eth + +ipset enp6s0 192.168.2.85 +ipset enp9s0 #192.168.2.95 +ipset enp9s0d1 192.168.35.85 +ipset enp10s0 #192.168.2.105 +ipset enp10s0d1 192.168.45.85 diff --git a/ip-switch/ipecamera6.sh b/ip-switch/ipecamera6.sh new file mode 100755 index 0000000..aab025d --- /dev/null +++ b/ip-switch/ipecamera6.sh @@ -0,0 +1,8 @@ +. functions.sh + +mlxconf eth + +ipset enp5s0 192.168.36.86 +ipset enp5s0d1 192.168.2.86 +ipset enp10s0 192.168.46.86 +ipset enp10s0d1 #192.168.2.96 diff --git a/ip/functions.sh b/ip/functions.sh deleted file mode 100644 index f912a6e..0000000 --- a/ip/functions.sh +++ /dev/null @@ -1,51 +0,0 @@ -function mlxreload { - rmmod ib_ucm - rmmod rdma_ucm - rmmod ib_uverbs - modprobe ib_uverbs disable_raw_qp_enforcement=1 - modprobe ib_ucm - modprobe rdma_ucm - modprobe mlx4_core fast_drop=1 log_num_mgm_entry_size=-1 - modprobe mlx4_en -} - -function mlxload { - reload=0 - - param=$(cat /sys/module/mlx4_core/parameters/log_num_mgm_entry_size) - [ $param -ne -1 ] && reload=1 - - [ $reload -eq 1 ] && mlxreload -} - -function portset { - pci=$1 - mode=$2 - ./connectx_port_config -d "$pci" -c "$mode,$mode" &> /dev/null -} - -function mlxconf { - mode=$1 - - mlxload - - pci=$(./connectx_port_config -s | grep PCI | grep -Po "[\da-f]{4}:[\da-f]{2}:[\da-f]{2}\.[\da-f]") - for dev in $pci; do - echo "Configuring: $dev" - portset $dev $mode - done -} - -function ipset { - int="$1" - ip="$2" - - if [ -n "$ip" ]; then - ip link set "$int" up - ip addr flush "$int" - ip addr show dev "$int" | grep "$ip" || ip addr add "$ip/24" dev "$int" - else - ip addr flush "$int" - ip link set "$int" down - fi -} diff --git a/ip/ipecamera0.sh b/ip/ipecamera0.sh deleted file mode 100755 index 06d16ce..0000000 --- a/ip/ipecamera0.sh +++ /dev/null @@ -1,7 +0,0 @@ -. ip/functions.sh - -mlxconf eth - -ipset enp4s0 192.168.2.80 -ipset enp8s0 #192.168.2.90 -ipset enp9s0 #192.168.2.100 diff --git a/ip/ipecamera3.sh b/ip/ipecamera3.sh deleted file mode 100755 index f003a1a..0000000 --- a/ip/ipecamera3.sh +++ /dev/null @@ -1,11 +0,0 @@ -. ip/functions.sh - -mlxconf eth - -ipset ens1 192.168.36.83 -ipset ens1d1 192.168.2.83 -ipset ens11 192.168.34.83 -ipset ens11d1 192.168.35.83 - -ipset enp6s0f0 #192.168.2.93 -ipset enp7s0f0 #192.168.2.103 diff --git a/ip/ipecamera4.sh b/ip/ipecamera4.sh deleted file mode 100755 index 81b1e18..0000000 --- a/ip/ipecamera4.sh +++ /dev/null @@ -1,11 +0,0 @@ -. ip/functions.sh - -mlxconf eth - -ipset ens1 192.168.34.84 -ipset ens1d1 192.168.2.84 -ipset ens11 192.168.46.84 -ipset ens11d1 192.168.45.84 -ipset enp6s0f0 #192.168.2.94 -ipset enp7s0f0 #192.168.2.104 - diff --git a/ip/ipecamera5.sh b/ip/ipecamera5.sh deleted file mode 100755 index f6660de..0000000 --- a/ip/ipecamera5.sh +++ /dev/null @@ -1,9 +0,0 @@ -. ip/functions.sh - -mlxconf eth - -ipset enp6s0 192.168.2.85 -ipset enp9s0 #192.168.2.95 -ipset enp9s0d1 192.168.35.85 -ipset enp10s0 #192.168.2.105 -ipset enp10s0d1 192.168.45.85 diff --git a/ip/ipecamera6.sh b/ip/ipecamera6.sh deleted file mode 100755 index dd41e1b..0000000 --- a/ip/ipecamera6.sh +++ /dev/null @@ -1,8 +0,0 @@ -. ip/functions.sh - -mlxconf eth - -ipset enp5s0 192.168.36.86 -ipset enp5s0d1 192.168.2.86 -ipset enp10s0 192.168.46.86 -ipset enp10s0d1 #192.168.2.96 diff --git a/update.sh b/update.sh index 19edbec..b8e00ed 100644 --- a/update.sh +++ b/update.sh @@ -1,7 +1,9 @@ lspci -d 15b3: mstflint -d 05:00.0 q -# MT_1090120019 (PCIe gen3, FDR, Eth 40Gb/s) +# MT_1090120019 (PCIe gen3, 2xFDR, Eth 40Gb/s) #mstflint -d 0a:00.0 -i fw/fw-ConnectX3-rel-2_42_5000-MCX354A-FCB_A2-A5-FlexBoot-3.4.752.bin burn -# MT_1090110028 (PCIe gen3, FDR, Eth 10Gb/s) -#mstflint -d 0a:00.0 -i fw/fw-ConnectX3-rel-2_42_5000-MCX354A-TCB_Ax-FlexBoot-3.4.752.bin \ No newline at end of file +# MT_1090110018 (PCIe gen3, 2xQDR, Eth 10Gb/s) +#mstflint -d 0a:00.0 -i fw-ConnectX3-rel-2_42_5000-MCX354A-QCB_Ax-FlexBoot-3.4.752.bin burn +# MT_1090110028 (PCIe gen3, 2xFDR, Eth 10Gb/s) +#mstflint -d 0a:00.0 -i fw/fw-ConnectX3-rel-2_42_5000-MCX354A-TCB_Ax-FlexBoot-3.4.752.bin burn -- cgit v1.2.3