+This is not intended to be a comprehensive "real world" test, however it is an apples to apples test using an NFS mount and a simple [https://gist.github.com/spiderr/404ed58a68d8b5064833968e975320eb|dd speed test] to compare InfiniBand and Ethernet in a virtualized and non-virtualized environment. |
+ |
+!NFS Export |
+{code source=bash}[root@guest1 ~]# cat /etc/exports |
+/master1 192.168.72.0/22(rw,insecure,async,all_squash,no_root_squash) 172.16.0.0/22(rw,insecure,async,all_squash,no_root_squash) |
+[root@baremetal2 ~]# nfsstat -m |
+/mnt/56ib from 172.16.0.41:/master1 |
+ Flags: rw,relatime,vers=4.1,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.16.0.14,local_lock=none,addr=172.16.0.41{/code} |
+ |
+ |
+{| class="table table-bordered table-condensed" |
+|+ Infiniband vs Ethernet Performance Stats, Results are all Per Second |
+|- class="text-center" |
+| colspan="2" | Connection |
+| colspan="3" | 40Gb IB Mellanox/EMC SX6005 |
+| colspan="3" | 40Gb Eth Cisco NX3064 |
+| colspan="3" | 10Gb Eth Cisco NX3064 |
+|- class="text-center" |
+| colspan="2" | |
+| colspan="3" | 16K * 64KB |
+| colspan="3" | 16K * 64KB |
+| colspan="3" | 16K * 64KB |
+|- class="text-center" |
+! Server !! Client !! Write !! Read !! w/Cache !! Write !! Read !! w/Cache !! Write !! Read !! w/Cache |
+|- class="text-right" |
+! Baremetal !! Baremetal || 3.2Gb/s 409MB/s || 9.8Gb/s 1227MB/s || 47.8Gb/s 5984MB/s || || || || 5.6Gb/s 700MB/s || 8.69Gb/s 1086MB/s || 49.4Gb/s 6184MB/s |
+|- |
+|- class="text-right" |
+! KVM SR-IOV !! Baremetal || 4.2Gb/s 529MB/s || 9.6Gb/s 1206MB/s || 47.2Gb/s 5910MB/s || 5.5Gb/s 689MB/s || 9.7Gb/s 1216MB/s || 44.8Gb/s 5602MB/s || 5.0Gb/s 631MB/s || 8.7Gb/s 1088MB/s || 46.4Gb/s 5802MB/s |
+|- |
+|- class="text-right" |
+!KVM SR-IOV !! KVM SR-IOV || 4.5Gb/s 659MB/s || 7.1Gb/s 893MB/s || 40.1Gb/s 5014MB/s || 6.5Gb/s 816MB/s || 7.0Gb/s 884MB/s || 45.3Gb/s 5669MB/s || || || |
+|- |
+|- class="text-right" |
+!KVM Bridge !! KVM Bridge || || || || 6.6Gb/s 8.36MB/s || 7.9Gb/s 997MB/s || 47.8Gb/s 5987MB/s || || || |
+|- |
+|- class="text-right" |
+| colspan=11 | |
+|- |
+|- class="text-right" |
+| colspan=2 | Baremetal, direct mount (max possible) || 7.85Gb/s 981MB/s || 19.1Gb/s 2390MB/s || 45.1Gb/s 5641MB/s || |
+|- |
+|} |
|
!Hardware
|
-Server: Dell r730xd, Dual Intel(R) Xeon(R) CPU E5-2620 v3 @ 2.40GHz
|
+!! NFS Client |
+Dell PowerEdge R610 Dual Intel(R) Xeon(R) X5675 @ 3.07GHz |
+Network Adapter 40Gb: Mellanox ConnectX-3 CX354A MT27500 Family ConnectX-3 |
+Network Adapter 10GbE: Mellanox ConnectX-2 =MT26448 ConnectX EN 10GigE, PCIe 2.0 5GT/s |
+ |
+!! NFS Server |
+Dell r730xd, Dual Intel(R) Xeon(R) CPU E5-2620 v3 @ 2.40GHz |
Network Adapter 40Gb: Mellanox ConnectX-3 CX354A MT27500 Family ConnectX-3
|
Network Adapter 10GbE: Mellanox ConnectX-2 =MT26448 ConnectX EN 10GigE, PCIe 2.0 5GT/s
|
-Storage: 9x12TB Seagate Exos 12Gb/S SAS 7200 RPM, PERC H730P
|
+Storage: 9x12TB Seagate Exos 12Gb/S SAS 7200 RPM, PERC H730P, mkfs.xfs -f -i size=512 -d su=64k,sw=8 /dev/sdb |
|
{code source=bash}[root@hyper0 ~]# omreport storage controller controller=0
|
Controller PERC H730P Mini(Embedded)
|