Difference between revisions of "CSIT/csit-perf-tc-naming-change"

From fd.io
Jump to: navigation, search
m (→‎Convention)
 
(22 intermediate revisions by 2 users not shown)
Line 33: Line 33:
 
#* '''<span style="color: maroon">10ge2p1x520</span>-<span style="color: red">ethip4</span>-<span style="color: blue">ip4base</span>-<span style="color: teal">iacldstbase</span>-<span style="color: purple">ndrdisc</span>.robot''' => 2 ports of 10GE on Intel x520 NIC, IPv4 baseline routed forwarding, ingress Access Control Lists baseline matching on destination, NDR throughput discovery.
 
#* '''<span style="color: maroon">10ge2p1x520</span>-<span style="color: red">ethip4</span>-<span style="color: blue">ip4base</span>-<span style="color: teal">iacldstbase</span>-<span style="color: purple">ndrdisc</span>.robot''' => 2 ports of 10GE on Intel x520 NIC, IPv4 baseline routed forwarding, ingress Access Control Lists baseline matching on destination, NDR throughput discovery.
 
#* '''<span style="color: maroon">40ge2p1vic1385</span>-<span style="color: red">ethip4</span>-<span style="color: blue">ip4base</span>-<span style="color: purple">ndrdisc</span>.robot''' => 2 ports of 40GE on Cisco vic1385 NIC, IPv4 baseline routed forwarding, NDR throughput discovery.
 
#* '''<span style="color: maroon">40ge2p1vic1385</span>-<span style="color: red">ethip4</span>-<span style="color: blue">ip4base</span>-<span style="color: purple">ndrdisc</span>.robot''' => 2 ports of 40GE on Cisco vic1385 NIC, IPv4 baseline routed forwarding, NDR throughput discovery.
−
#* '''<span style="color: maroon">virtio2p</span>-<span style="color: red">ethip4</span>-<span style="color: blue">ip4base</span>-<span style="color: purple">func</span>.robot''' => 2 ports of Virtio, IPv4 baseline routed forwarding, functional tests.
+
#* '''<span style="color: maroon">eth2p</span>-<span style="color: red">ethip4</span>-<span style="color: blue">ip4base</span>-<span style="color: purple">func</span>.robot''' => 2 ports of Ethernet, IPv4 baseline routed forwarding, functional tests.
 
# '''<u>Physical port to VM (or VM chain) to physical port - a.k.a. NIC2VM2NIC, P2V2P, NIC2VMchain2NIC, P2V2V2P</u>'''
 
# '''<u>Physical port to VM (or VM chain) to physical port - a.k.a. NIC2VM2NIC, P2V2P, NIC2VMchain2NIC, P2V2V2P</u>'''
 
#* '''<span style="color: maroon">PortNICConfig</span>-<span style="color: red">WireEncapsulation</span>-<span style="color: blue">PacketForwardingFunction</span>-<span style="color: teal">PacketProcessingFunction1</span>-...-<span style="color: teal">PacketProcessingFunctionN</span>-<span style="color: navy">VirtEncapsulation</span>-<span style="color: olive">VirtPortConfig</span>-<span style="color: fuchsia">VMconfig</span>-<span style="color: purple">TestType</span>'''
 
#* '''<span style="color: maroon">PortNICConfig</span>-<span style="color: red">WireEncapsulation</span>-<span style="color: blue">PacketForwardingFunction</span>-<span style="color: teal">PacketProcessingFunction1</span>-...-<span style="color: teal">PacketProcessingFunctionN</span>-<span style="color: navy">VirtEncapsulation</span>-<span style="color: olive">VirtPortConfig</span>-<span style="color: fuchsia">VMconfig</span>-<span style="color: purple">TestType</span>'''
Line 39: Line 39:
 
#* '''<span style="color: maroon">10ge2p1x520</span>-<span style="color: red">ethip4vxlan</span>-<span style="color: blue">l2bdbasemaclrn</span>-<span style="color: navy">eth</span>-<span style="color: olive">2vhost</span>-<span style="color: fuchsia">1vm</span>-<span style="color: purple">ndrdisc</span>.robot''' => 2 ports of 10GE on Intel x520 NIC, IPv4 VXLAN Ethernet, L2 bridge-domain switching to/from two vhost interfaces and one VM, NDR throughput discovery.
 
#* '''<span style="color: maroon">10ge2p1x520</span>-<span style="color: red">ethip4vxlan</span>-<span style="color: blue">l2bdbasemaclrn</span>-<span style="color: navy">eth</span>-<span style="color: olive">2vhost</span>-<span style="color: fuchsia">1vm</span>-<span style="color: purple">ndrdisc</span>.robot''' => 2 ports of 10GE on Intel x520 NIC, IPv4 VXLAN Ethernet, L2 bridge-domain switching to/from two vhost interfaces and one VM, NDR throughput discovery.
 
#* '''<span style="color: maroon">10ge2p1x520</span>-<span style="color: red">ethip4vxlan</span>-<span style="color: blue">l2bdbasemaclrn</span>-<span style="color: navy">eth</span>-<span style="color: olive">4vhost</span>-<span style="color: fuchsia">2vm</span>-<span style="color: purple">ndrdisc</span>.robot''' => 2 ports of 10GE on Intel x520 NIC, IPv4 VXLAN Ethernet, L2 bridge-domain switching to/from four vhost interfaces and two VMs, NDR throughput discovery.
 
#* '''<span style="color: maroon">10ge2p1x520</span>-<span style="color: red">ethip4vxlan</span>-<span style="color: blue">l2bdbasemaclrn</span>-<span style="color: navy">eth</span>-<span style="color: olive">4vhost</span>-<span style="color: fuchsia">2vm</span>-<span style="color: purple">ndrdisc</span>.robot''' => 2 ports of 10GE on Intel x520 NIC, IPv4 VXLAN Ethernet, L2 bridge-domain switching to/from four vhost interfaces and two VMs, NDR throughput discovery.
−
#* '''<span style="color: maroon">virtio2p</span>-<span style="color: red">ethip4vxlan</span>-<span style="color: blue">l2bdbasemaclrn</span>-<span style="color: navy">eth</span>-<span style="color: olive">2vhost</span>-<span style="color: fuchsia">1vm</span>-<span style="color: purple">func</span>.robot''' => 2 ports of Virtio, IPv4 VXLAN Ethernet, L2 bridge-domain switching to/from two vhost interfaces and one VM, functional tests.
+
#* '''<span style="color: maroon">eth2p</span>-<span style="color: red">ethip4vxlan</span>-<span style="color: blue">l2bdbasemaclrn</span>-<span style="color: navy">eth</span>-<span style="color: olive">2vhost</span>-<span style="color: fuchsia">1vm</span>-<span style="color: purple">func</span>.robot''' => 2 ports of Ethernet, IPv4 VXLAN Ethernet, L2 bridge-domain switching to/from two vhost interfaces and one VM, functional tests.
 +
# '''<u>API CRUD tests - Create (Write), Read (Retrieve), Update (Modify), Delete (Destroy) operations for configuration and operational data</u>'''
 +
#* '''<span style="color: maroon">ManagementTestKeyword</span>-<span style="color: red">ManagementOperation</span>-<span style="color: blue">ManagedFunction1</span>-...-<span style="color: blue">ManagedFunctionN</span>-<span style="color: teal">ManagementAPI1</span>-<span style="color: teal">ManagementAPIN</span>-<span style="color: purple">TestType</span>'''
 +
#* '''<span style="color: maroon">mgmt</span>-<span style="color: red">cfg</span>-<span style="color: blue">lisp</span>-<span style="color: teal">apivat</span>-<span style="color: purple">func</span>''' => configuration of LISP with VAT API calls, functional tests.
 +
#* '''<span style="color: maroon">mgmt</span>-<span style="color: red">cfg</span>-<span style="color: blue">l2bd</span>-<span style="color: teal">apihc</span>-<span style="color: teal">apivat</span>-<span style="color: purple">func</span>''' => configuration of L2 Bridge-Domain with HoneyComb API and VAT API calls, functional tests.
 +
#* '''<span style="color: maroon">mgmt</span>-<span style="color: red">oper</span>-<span style="color: blue">int</span>-<span style="color: teal">apihcnc</span>-<span style="color: purple">func</span>''' => reading status and operational data of interface with HoneyComb NetConf API calls, functional tests.
 +
#* '''<span style="color: maroon">mgmt</span>-<span style="color: red">cfg</span>-<span style="color: blue">int</span>-<span style="color: blue">tap</span>-<span style="color: teal">apihcnc</span>-<span style="color: purple">func</span>''' => configuration of tap interfaces with HoneyComb NetConf API calls, functional tests.
 +
#* '''<span style="color: maroon">mgmt</span>-<span style="color: red">notif</span>-<span style="color: blue">int</span>-<span style="color: blue">subint</span>-<span style="color: teal">apihcnc</span>-<span style="color: purple">func</span>''' => notifications of interface and sub-interface events with HoneyComb NetConf Notifications, functional tests.
  
 
===Legend===
 
===Legend===
 
# '''<span style="color: maroon">PortNICConfig</span>''' - packet network port configuration: physical port type, NIC model, port-NIC locality
 
# '''<span style="color: maroon">PortNICConfig</span>''' - packet network port configuration: physical port type, NIC model, port-NIC locality
−
## '''<n>ge''' | '''virtio''' - port type
+
## '''<n>ge''' | '''eth''' - port type
 
##* '''<n>ge''' - Gigabit Ethernet, n=10|40|100|200|400.
 
##* '''<n>ge''' - Gigabit Ethernet, n=10|40|100|200|400.
−
##* '''virtio''' - virtio.
+
##* '''eth''' - generic Ethernet interface.
 
## '''<n>p''' - number of physical Ethernet ports, ports distributed evenly across tested NICs, n=1,2,...
 
## '''<n>p''' - number of physical Ethernet ports, ports distributed evenly across tested NICs, n=1,2,...
 
## '''<n><nic_model>''' - number of NICs and NIC model, n=1,2,...
 
## '''<n><nic_model>''' - number of NICs and NIC model, n=1,2,...
 +
# '''<span style="color: maroon">ManagementTestKeyword</span>''' - prefix for management API tests, no packet forwarding.
 +
## '''mgmt''' - management operations test.
 
# '''<span style="color: red">WireEncapsulation</span>'''
 
# '''<span style="color: red">WireEncapsulation</span>'''
 
## '''eth''' | '''dot1q''' | '''dot1ad''' - Ethernet header
 
## '''eth''' | '''dot1q''' | '''dot1ad''' - Ethernet header
Line 53: Line 62:
 
##* '''dot1q''' - single VLAN tag on physical wire.
 
##* '''dot1q''' - single VLAN tag on physical wire.
 
##* '''dot1ad''' - double VLAN tag on physical wire.
 
##* '''dot1ad''' - double VLAN tag on physical wire.
 +
##* '''--''' - OR separator, if present multiple encapsulations present on interfaces.
 
## '''ip4''' | '''ip6''' | '''ip6e''' - outer IP header
 
## '''ip4''' | '''ip6''' | '''ip6e''' - outer IP header
 
##* '''ip4''' - IPv4 header.
 
##* '''ip4''' - IPv4 header.
 
##* '''ip6''' - IPv6 header.
 
##* '''ip6''' - IPv6 header.
 
##* '''ip6e''' - IPv6 extended header.
 
##* '''ip6e''' - IPv6 extended header.
 +
##* '''ip6ip4''' - IPv4-over-IPv6 softwire header.
 +
##* '''--''' - OR separator, if present multiple encapsulations present on interfaces.
 +
## '''ipsectrt''' | '''ipsectnl''' - IPSec crypto
 +
##* '''ipsectrt''' - IPSec transport mode.
 +
##* '''ipsectnl''' - IPSec tunnel mode.
 
## '''vxlan''' | '''lisp''' | '''gre''' | '''gpe''' - tunnel header
 
## '''vxlan''' | '''lisp''' | '''gre''' | '''gpe''' - tunnel header
 
##* '''vxlan''' - VXLAN tunnel header.
 
##* '''vxlan''' - VXLAN tunnel header.
Line 62: Line 77:
 
##* '''gre''' - GRE tunnel header.
 
##* '''gre''' - GRE tunnel header.
 
##* '''gpe''' - GPE tunnel header.
 
##* '''gpe''' - GPE tunnel header.
 +
# '''<span style="color: red">ManagementOperation</span>'''
 +
## '''cfg''' | '''oper''' | '''notif''' | '''statepersist''' - management operation type
 +
##* '''cfg''' - configuration writes, modifies, deletes and read verification.
 +
##* '''oper''' - operational state reads.
 +
##* '''notif''' - notifications generation on events.
 +
##* '''statepersist''' - configuration and operation state persistence.
 
# '''<span style="color: blue">PacketForwardingFunction</span>'''
 
# '''<span style="color: blue">PacketForwardingFunction</span>'''
 
## '''l2bd''' - VPP L2 bridge-domain
 
## '''l2bd''' - VPP L2 bridge-domain
Line 82: Line 103:
 
###* '''base''' - baseline test with one IPv6 flow received per interface.
 
###* '''base''' - baseline test with one IPv6 flow received per interface.
 
###* '''scale<n>''' - scale tests with many IPv6 flows received per interface, <n> - number of /128 IPv6 address entries present in IPv6 lookup table.
 
###* '''scale<n>''' - scale tests with many IPv6 flows received per interface, <n> - number of /128 IPv6 address entries present in IPv6 lookup table.
 +
# '''<span style="color: blue">ManagedFunction</span>'''
 +
## '''int''' - interface.
 +
## '''subint''' - sub-interface.
 +
## '''inttap''' - tap interface.
 +
## '''intip4''' - IPv4 interface.
 +
## '''intip6''' - IPv6 interface.
 +
## '''l2bd''' - L2 Bridge-Domain.
 +
## '''acl''' - Access Control List.
 +
## '''ietfacl''' - IETF-style Access Control List.
 +
## '''spanrx''' - Switched Port Analyzer (SPAN) port mirroring of Rx'ed packets.
 +
## '''l2fib''' - L2 Forwarding Information Base a.k.a. MAC lookup and forwarding table.
 +
## '''snat44''' - static Network Address Translation (NAT) for IPv4 to IPv4.
 +
## '''nsh''' - Network Service Header encapsulation functions.
 +
## '''vxlangpe''' - VXLAN GPE encapsulation functions.
 +
## '''pbb''' - PBB Provider Backbone Bridge encapsulation functions.
 
# '''<span style="color: teal">PacketProcessingFunction</span>'''
 
# '''<span style="color: teal">PacketProcessingFunction</span>'''
 
## '''l2shg''' - L2 Split Horizon Group.
 
## '''l2shg''' - L2 Split Horizon Group.
 +
## '''vlantrans''' - VLAN-ID tag translation
 +
### '''11''' | '''12''' | '''21''' | '''22'''
 +
###* '''11''' - one-to-one VLAN-ID tag translation, dot1q to dot1q.
 +
###* '''12''' - one-to-two VLAN-ID tag translation, dot1q to dot1ad.
 +
###* '''21''' - two-to-one VLAN-ID tag translation, dot1ad to dot1q.
 +
###* '''22''' - two-to-two VLAN-ID tag translation, dot1ad to dot1ad.
 
## '''iacldstbase''' - ingress Access Control Lists baseline matching on destination.
 
## '''iacldstbase''' - ingress Access Control Lists baseline matching on destination.
 
## '''ipolicemarkbase''' - ingress policer and marker baseline.
 
## '''ipolicemarkbase''' - ingress policer and marker baseline.
 
## '''copwhtlistbase''' - ingress IP Cop security whitelist filter entries, /24 for IPv4, /64 for IPv6.
 
## '''copwhtlistbase''' - ingress IP Cop security whitelist filter entries, /24 for IPv4, /64 for IPv6.
 +
## '''copblklistbase''' - ingress IP Cop security blacklist filter entries, /24 for IPv4, /64 for IPv6
 +
## '''rpf''' - IP Reverse Path Forwarding check.
 +
## '''swirelw46''' - softwire IPv4-over-IPv6 tunnel concentrator, LightWeight-IPv4-over-IPv6, LW46.
 +
## '''swiremape''' - softwire IPv4-over-IPv6 tunnel concentrator, MAP-E.
 +
## '''swiremapt''' - softwire IPv4-over-IPv6 tunnel concentrator, MAP-T.
 +
## '''ip4ipfix''' - IPv4 IPFIX flow statistics
 +
### '''base''' | '''scale'''
 +
###* '''base''' - baseline tests with one IPv4 flow received per interface.
 +
###* '''scale''' - scale tests with many IPv4 flows received per interface.
 +
## '''ip6ipfix''' - IPv6 IPFIX flow statistics
 +
### '''base''' | '''scale'''
 +
###* '''base''' - baseline tests with one IPv6 flow received per interface.
 +
###* '''scale''' - scale tests with many IPv6 flows received per interface.
 +
## '''spanrx''' - Switched Port Analyzer (SPAN) port mirroring of Rx'ed packets, receive direction.
 +
## '''spantx''' - Switched Port Analyzer (SPAN) port mirroring of Tx'ed packets, transmit direction.
 +
## '''ip4dhcpclient''' - IPv4 DHCP client.
 +
## '''ip4dhcpcproxy''' - IPv4 DHCP proxy.
 +
# '''<span style="color: teal">ManagementAPI</span>'''
 +
## '''apihc''' - HoneyComb API.
 +
## '''apihcnc''' - HoneyComb NetConf API.
 +
## '''apivat''' - "VPP API Test" (VAT) API.
 
# '''<span style="color: navy">VirtEncapsulation</span>'''
 
# '''<span style="color: navy">VirtEncapsulation</span>'''
 
## '''eth''' | '''dot1q''' | '''dot1ad'''
 
## '''eth''' | '''dot1q''' | '''dot1ad'''
Line 94: Line 157:
 
# '''<span style="color: olive">VirtPortConfig</span>'''
 
# '''<span style="color: olive">VirtPortConfig</span>'''
 
## '''<n>vhost''' - vhost virtual interface, n=1,2, ...
 
## '''<n>vhost''' - vhost virtual interface, n=1,2, ...
 +
## '''<n>tap''' - tap virtual interface, n=1,2, ...
 
# '''<span style="color: fuchsia">VMconfig</span>'''
 
# '''<span style="color: fuchsia">VMconfig</span>'''
 
## '''<n>vm''' - Virtual Machines, n=1,2, ...
 
## '''<n>vm''' - Virtual Machines, n=1,2, ...
Line 103: Line 167:
 
##* '''pdrchk''' - PDR throughput check against a reference NDR throughput.
 
##* '''pdrchk''' - PDR throughput check against a reference NDR throughput.
 
##* '''func''' - functional tests.
 
##* '''func''' - functional tests.
 +
 +
===Examples===
 +
 +
====CSIT Perf - Suites and Test Cases====
 +
Below examples of CSIT Performance Suite Names (filename.robot) and included test cases.
 +
 +
# suite: 10ge2p1x520-ethip4-ip4base-ndrdisc.robot
 +
## test cases:
 +
### tc01-64B-2t1c-ethip4-ip4base-ndrdisc
 +
### tc02-IMIX-1t1c-ethip4-ip4base-ndrdisc
 +
### tc03-1518B-1t1c-ethip4-ip4base-ndrdisc
 +
# suite: 10ge2p1x710-ethip4-ip4base-ndrdisc.robot
 +
## test cases:
 +
### tc01-64B-2t1c-ethip4-ip4base-ndrdisc
 +
### tc02-IMIX-1t1c-ethip4-ip4base-ndrdisc
 +
### tc03-1518B-1t1c-ethip4-ip4base-ndrdisc
 +
# suite: 40ge2p1xl710-ethip4-ip4base-ndrdisc.robot
 +
## test cases:
 +
### tc01-64B-2t1c-ethip4-ip4base-ndrdisc
 +
### tc02-IMIX-1t1c-ethip4-ip4base-ndrdisc
 +
### tc03-1518B-1t1c-ethip4-ip4base-ndrdisc
 +
 +
====CSIT Perf - Trending Graphs====
 +
Below examples of CSIT Perf Trending Graphs and included test case result plots.
 +
 +
# graph: 64B-1t1c-ethip4-ip4*-ndrdisc
 +
## test case result plots:
 +
### 10ge2p1x520-ethip4-ip4base
 +
### 10ge2p1x710-ethip4-ip4base
 +
### 40ge2p1xl710-ethip4-ip4base
 +
### 10ge2p1x520-ethip4-ip4scale20k
 +
### 10ge2p1x520-ethip4-ip4scale200k
 +
### 10ge2p1x520-ethip4-ip4scale2m
 +
### 10ge2p1x520-ethip4-ip4base-copwhtlistbase
 +
### 10ge2p1x520-ethip4-ip4base-iacldstbase
 +
### 10ge2p1x520-ethip4-ip4base-ipolicemarkbase
 +
# graph: 64B-1t1c-ethip6-ip6*-ndrdisc
 +
## test case result plots:
 +
### 10ge2p1x520-ethip6-ip6base
 +
### 10ge2p1x710-ethip6-ip6base
 +
### 40ge2p1xl710-ethip6-ip6base
 +
### 10ge2p1x520-ethip6-ip6scale20k
 +
### 10ge2p1x520-ethip6-ip6scale200k
 +
### 10ge2p1x520-ethip6-ip6scale2m
 +
### 10ge2p1x520-ethip6-ip6base-copwhtlistbase
 +
### 10ge2p1x520-ethip6-ip6base-iacldstbase
 +
### 10ge2p1x520-ethip6-ip6base-ipolicemarkbase
  
 
==CSIT Performance Tests==
 
==CSIT Performance Tests==
Line 332: Line 443:
 
|}
 
|}
  
−
===Trending Graphs Tagging===
+
===Performance Trending Graphs===
−
In order to group performance TC results into performance trending graphs, CSIT is using RF tags. CSIT RF tags are described in `/docs/tag_documentation.rst'.
+
One of the CSIT goals is to auto-generate VPP performance trending graphs for tested use cases and configurations. Trending graphs are auto-generated for Jenkins csit-perf* jobs using Jenkins plugin <add link> and for CSIT release reports using separate python scripts <add link>.
−
RF tag changes related to performance TC naming change are captured as part of [https://gerrit.fd.io/r/#/c/3856/12/docs/tag_documentation.rst patch #3856].
+
For better readibility VPP performance results are marked with RF tags and grouped into separate graphs as follows:
 +
 
 +
# L2BD, L2XC
 +
#* tags: L2BDMACSTAT or L2BDMACLRN or L2XCFWD
 +
# IPv4 base, scale, feature
 +
#* tags: IP4FWD
 +
# IPv6 base, scale, feature
 +
#* tags: IP6FWD
 +
# Overlay tunnels - LISP, VXLAN, GPE, GRE
 +
#* tags: VXLAN or LISP or LISPGPE or VXLANGPE or GRE
 +
# Crypto in software: IP4FWD, IP6FWD
 +
#* tags: IPSECSW and (IPSECTRAN or IPSECTUN)
 +
# Crypto in hardware: IP4FWD, IP6FWD
 +
#* tags: IPSECHW and (IPSECTRAN or IPSECTUN)
 +
# Overlay tunnels with crypto in software
 +
#* tags: (VXLAN or LISP or LISPGPE or VXLANGPE or GRE) and IPSECSW and (IPSECTRAN or IPSECTUN)
 +
# Overlay tunnels with crypto in hardware
 +
#* tags: (VXLAN or LISP or LISPGPE or VXLANGPE or GRE) and IPSEHW and (IPSECTRAN or IPSECTUN)
 +
# vhost-user
 +
#* tags: VHOST and (ETH or DOT1Q or VXLAN)
 +
 
 +
For each of the above, separate graph series is auto-generated for:
 +
 
 +
# Throughput type
 +
## NDR
 +
##* tags: NDRDISC
 +
## PDR
 +
##* tags: PDRDISC
 +
# Worker thread and core configuration
 +
## 1t1c - 1 worker-thread, 1 core
 +
##* tags: 1T1C
 +
## 2t2c - 2 worker-threads, 2 cores
 +
##* tags: 2T2C
 +
## 4t4c - 4 worker-threads, 4 cores
 +
##* tags: 2T2C
 +
 
 +
Resulting total number of auto-generated graphs: 9*5 = 45 graphs.
 +
 
 +
===Test RF Tagging for Trending Graphs===
 +
In order to group performance test results into auto-generated trending graphs, CSIT is using Robot Framework (RF) tags. CSIT RF tags are defined and described in `/docs/tag_documentation.rst'.
 +
RF tag changes related to test naming change are captured in [https://gerrit.fd.io/r/#/c/3856/12/docs/tag_documentation.rst patch #3856].
 +
 
 +
# '''<u>Functional and performance tags</u>'''
 +
## '''FUNCTEST''' - All functional test cases.
 +
## '''PERFTEST''' - All performance test cases.
 +
# '''<u>Performance testing tags</u>'''
 +
## '''PDRDISC''' - Partial Drop Rate evaluation of single run result, with non-zero packet loss tolerance (LT) expressed in percentage of packets transmitted
 +
## '''NDRDISC''' - Non Drop Rate evaluation of results. Loss acceptance of dropped packets is set to zero lost packets
 +
## '''NDRCHK''' - Performance tests where TG verifies DUTs' throughput at ref-NDR (reference Non Drop Rate) with zero packet loss tolerance
 +
## '''NDRPDRDISC''' - Find performance of DUT based on RFC2544 with linear / binary / combined search. (Previous LONG tests.)
 +
# '''<u>Test type tags</u>'''
 +
## '''BASE''' - Baseline test cases, no encapsulation, no feature(s) configured in tests.
 +
## '''SCALE''' - Scale test cases.
 +
## '''ENCAP''' - Test cases where encapsulation is used. Use also encapsulation tag(s).
 +
## '''FEATURE''' - At least one feature is configured in test cases. Use also feature tag(s).
 +
# '''<u>Scaling tags</u>'''
 +
## '''FIB_20K''' - 2x10,000 entries in single fib table
 +
## '''FIB_200K''' - 2x100,000 entries in single fib table
 +
## '''FIB_2M''' - 2x1,000,000 entries in single fib table
 +
# '''<u>Forwarding mode tags</u>'''
 +
## '''L2BDMACSTAT''' - VPP L2 bridge-domain, L2 MAC static.
 +
## '''L2BDMACLRN''' - VPP L2 bridge-domain, L2 MAC learning.
 +
## '''L2XCFWD''' - VPP L2 point-to-point cross-connect.
 +
## '''IP4FWD''' - VPP IPv4 routed forwarding.
 +
## '''IP6FWD''' - VPP IPv6 routed forwarding.
 +
# '''<u>Underlay tags</u>'''
 +
## '''IP4UNRLAY''' - IPv4 underlay.
 +
## '''IP6UNRLAY''' - IPv6 underlay.
 +
## '''MPLSUNRLAY''' - MPLS underlay.
 +
# '''<u>Overlay tags</u>'''
 +
## '''L2OVRLAY''' - L2 overlay.
 +
## '''IP4OVRLAY''' - IPv4 overlay (IPv4 payload).
 +
## '''IP6OVRLAY''' - IPv6 overlay (IPv6 payload).
 +
# '''<u>Encapsulation tags</u>'''
 +
## '''ETH''' - All test cases with base Ethernet (no encapsulation).
 +
## '''DOT1Q''' - All test cases with dot1q.
 +
## '''DOT1AD''' - All test cases with dot1ad.
 +
## '''LISP''' - All test cases with LISP.
 +
## '''LISPGPE''' - All test cases with LISP-GPE.
 +
## '''VXLAN''' - All test cases with Vxlan.
 +
## '''VXLANGPE''' - All test cases with VXLAN-GPE.
 +
## '''GRE''' - All test cases with GRE.
 +
# '''<u>Interface tags</u>'''
 +
## '''PHY''' - All test cases which use physical interface(s).
 +
## '''VHOST''' - All test cases which uses VHOST.
 +
## '''TUNTAP''' - All test cases which uses TUN and TAP.
 +
## '''AFPKT''' - All test cases which uses AFPKT.
 +
## '''NETMAP''' - All test cases which uses Netmap.
 +
# '''<u>Feature tags</u>'''
 +
## '''IACLDST''' - iACL destination.
 +
## '''COPWHLIST''' - COP whitelist.
 +
# '''<u>Client-workload tags</u>'''
 +
## '''VM''' - All test cases which use at least one virtual machine.
 +
## '''LXC''' - All test cases which use Linux container.
 +
## '''APP''' - All test cases with specific APP use.
 +
# '''<u>Multi-threading tags</u>'''
 +
## '''STHREAD''' - All test cases using single poll mode thread.
 +
## '''MTHREAD''' - All test cases using more then one poll mode driver thread.
 +
## '''1NUMA''' - All test cases with packet processing on single socket.
 +
## '''2NUMA''' - All test cases with packet processing on two sockets.
 +
## '''SMT''' - All test cases with symmetric Multi-Threading (HyperThreading) enabled.
 +
## '''NOSMT''' - All test cases with symmetric Multi-Threading (HyperThreading) disabled.
 +
## '''1T1C''' - 1 worker thread pinned to 1 dedicated physical core. 1 receive queue per interface. Main thread pinned to core 0.
 +
## '''2T2C''' - 2 worker threads pinned to 2 dedicated physical cores. 1 receive queue per interface. Main thread pinned to core 0.
 +
## '''4T4C''' - 4 worker threads pinned to 4 dedicated physical cores. 2 receive queues per interface. Main thread pinned to core 0.
 +
## '''6T6C''' - 6 worker threads pinned to 6 dedicated physical cores. 3 receive queues per interface. Main thread pinned to core 0.
 +
## '''8T8C''' - 8 worker threads pinned to 8 dedicated physical cores. 4 receive queues per interface. Main thread pinned to core 0.
  
 
==CSIT Functional Tests==
 
==CSIT Functional Tests==
 
Functional tests executed in FD.io CSIT VIRL testbeds.
 
Functional tests executed in FD.io CSIT VIRL testbeds.
−
Naming change coded in FD.io CSIT project by [<gerrit link to be added> patch #<gerrit patch id to be added>].
+
Functional tests naming change coded in FD.io CSIT project by [https://gerrit.fd.io/r/#/c/4753/ patch #4753].
  
 
Sorted alphabetically by new name.
 
Sorted alphabetically by new name.
Line 345: Line 562:
 
! #
 
! #
 
! New Test Filename
 
! New Test Filename
−
! Old Test Filename
+
! Old Test Filename and TC refs
 
|-
 
|-
 
! 1
 
! 1
−
| virtio2p-eth-l2bdbasemaclrn-func.robot
+
| eth2p-dot1ad--dot1q-l2bdbasemaclrn-vlantrans21-func.robot
−
| bridge_domain_untagged.robot
+
| vlan_tag_translate_l2_bridge_domain_ipv4.robot TC07,TC08
 
|-
 
|-
 
! 2
 
! 2
−
| virtio2p-eth-l2bdbasemacstc-func.robot
+
| eth2p-dot1ad--dot1q-l2bdbasemaclrn-vlantrans21-func.robot
−
| bridge_domain_untagged.robot
+
| vlan_tag_translate_l2_bridge_domain_ipv6.robot TC07,TC08
 
|-
 
|-
 
! 3
 
! 3
−
| virtio2p-eth-l2bdbasemaclrn-l2shg-func.robot
+
| eth2p-dot1ad--dot1q-l2xcbase-vlantrans21-func.robot
−
| bridge_domain_untagged.robot
+
| vlan_tag_translate_l2_xconnect_ipv4.robot TC07,TC08
 
|-
 
|-
 
! 4
 
! 4
−
| virtio2p-eth-l2bdbasemaclrn-eth-2vhost-1vm-func.robot
+
| eth2p-dot1ad--dot1q-l2xcbase-vlantrans21-func.robot
−
| bridge_domain_untagged.robot
+
| vlan_tag_translate_l2_xconnect_ipv6.robot TC07,TC08
 
|-
 
|-
 
! 5
 
! 5
−
| virtio2p-eth-l2bdbasemacstc-eth-2vhost-1vm-func.robot
+
| eth2p-dot1ad-l2bdbasemaclrn-vlantrans22-func.robot
−
| bridge_domain_untagged.robot
+
| vlan_tag_translate_l2_bridge_domain_ipv4.robot TC09..TC12
 +
|-
 +
! 6
 +
| eth2p-dot1ad-l2bdbasemaclrn-vlantrans22-func.robot
 +
| vlan_tag_translate_l2_bridge_domain_ipv6.robot TC09..TC12
 +
|-
 +
! 7
 +
| eth2p-dot1ad-l2xcbase-func.robot
 +
| qinq_l2_xconnect.robot TC01
 +
|-
 +
! 8
 +
| eth2p-dot1ad-l2xcbase-vlantrans22-func.robot
 +
| vlan_tag_translate_l2_xconnect_ipv4.robot TC09..TC12
 +
|-
 +
! 9
 +
| eth2p-dot1ad-l2xcbase-vlantrans22-func.robot
 +
| vlan_tag_translate_l2_xconnect_ipv6.robot TC09..TC12
 +
|-
 +
! 10
 +
| eth2p-dot1q--dot1ad-l2bdbasemaclrn-vlantrans12-func.robot
 +
| vlan_tag_translate_l2_bridge_domain_ipv4.robot TC03..TC06
 +
|-
 +
! 11
 +
| eth2p-dot1q--dot1ad-l2bdbasemaclrn-vlantrans12-func.robot
 +
| vlan_tag_translate_l2_bridge_domain_ipv6.robot TC03..TC06
 +
|-
 +
! 12
 +
| eth2p-dot1q--dot1ad-l2xcbase-vlantrans12-func.robot
 +
| vlan_tag_translate_l2_xconnect_ipv4.robot TC03..TC06
 +
|-
 +
! 13
 +
| eth2p-dot1q--dot1ad-l2xcbase-vlantrans12-func.robot
 +
| vlan_tag_translate_l2_xconnect_ipv6.robot TC03..TC06
 +
|-
 +
! 14
 +
| eth2p-dot1q-l2bdbasemaclrn-eth-4vhost-2vm-fds-provider-nets-func.robot
 +
| provider_network.robot TC01
 +
|-
 +
! 15
 +
| eth2p-dot1q-l2bdbasemaclrn-vlantrans11-ipfunc.robot
 +
| vlan_tag_translate_l2_bridge_domain_ipv4.robot TC01,TC02
 +
|-
 +
! 16
 +
| eth2p-dot1q-l2bdbasemaclrn-vlantrans11-ipfunc.robot
 +
| vlan_tag_translate_l2_bridge_domain_ipv6.robot TC01,TC02
 +
|-
 +
! 17
 +
| eth2p-dot1q-l2xcbase-vlantrans11-func.robot
 +
| vlan_tag_translate_l2_xconnect_ipv4.robot TC01,TC02
 +
|-
 +
! 18
 +
| eth2p-dot1q-l2xcbase-vlantrans11-func.robot
 +
| vlan_tag_translate_l2_xconnect_ipv6.robot TC01,TC02
 +
|-
 +
! 19
 +
| eth2p-dot1qip4vxlan-l2bdbasemaclrn-func.robot
 +
| vxlan_bd_dot1q.robot TC01
 +
|-
 +
! 21
 +
| eth2p-eth-l2bdbasemaclrn-eth-2tap-func.robot
 +
| tap_interface.robot TC01
 +
|-
 +
! 22
 +
| eth2p-eth-l2bdbasemaclrn-eth-2vhost-1vm-func.robot
 +
| bridge_domain_untagged.robot TC06,TC07
 +
|-
 +
! 23
 +
| eth2p-eth-l2bdbasemaclrn-func.robot
 +
| bridge_domain_untagged.robot TC01..TC03
 +
|-
 +
! 24
 +
| eth2p-eth-l2bdbasemaclrn-l2shg-eth-2tap-func.robot
 +
| tap_interface.robot TC04,TC05
 +
|-
 +
! 25
 +
| eth2p-eth-l2bdbasemaclrn-l2shg-func.robot
 +
| bridge_domain_untagged.robot TC05
 +
|-
 +
! 26
 +
| eth2p-eth-l2bdbasemacstc-eth-2vhost-1vm-func.robot
 +
| bridge_domain_untagged.robot TC08,TC09
 +
|-
 +
! 27
 +
| eth2p-eth-l2bdbasemacstc-func.robot
 +
| bridge_domain_untagged.robot TC04
 +
|-
 +
! 28
 +
| eth2p-eth-l2xcbase-eth-2vhost-1vm-func.robot
 +
| l2_xconnect_untagged.robot TC03,TC04
 +
|-
 +
! 29
 +
| eth2p-eth-l2xcbase-func.robot
 +
| l2_xconnect_untagged.robot TC01,TC02
 +
|-
 +
! 30
 +
| eth2p-eth-l2xcbase-iacldstbase-func.robot
 +
| l2_iacl_untagged.robot TC01..TC04
 +
|-
 +
! 31
 +
| eth2p-ethip4--ethip6ip4-ip4base--ip6base-swirelw46-func.robot
 +
| lightweight_4over6.robot TC01..TC04
 +
|-
 +
! 32
 +
| eth2p-ethip4--ethip6ip4-ip4base--ip6base-swiremape-func.robot
 +
| map_e.robot TC01..TC09
 +
|-
 +
! 33
 +
| eth2p-ethip4--ethip6ip4-ip4base--ip6base-swiremapt-func.robot
 +
| map_t.robot TC01
 +
|-
 +
! 34
 +
| eth2p-ethip4-ip4base-copblklistbase-func.robot
 +
| cop_whitelist_blacklist.robot TC02
 +
|-
 +
! 35
 +
| eth2p-ethip4-ip4base-copwhlistbase-func.robot
 +
| cop_whitelist_blacklist.robot TC01
 +
|-
 +
! 36
 +
| eth2p-ethip4-ip4base-eth-1tap-func.robot
 +
| tap_interface.robot TC02,TC03
 +
|-
 +
! 37
 +
| eth2p-ethip4-ip4base-func.robot
 +
| ipv4_untagged.robot TC01..TC08
 +
|-
 +
! 38
 +
| eth2p-ethip4-ip4base-iacldstbase-func.robot
 +
| ipv4_iacl_untagged.robot TC01..TC11
 +
|-
 +
! 39
 +
| eth2p-ethip4-ip4base-ip4arp-func.robot
 +
| ipv4_arp_untagged.robot TC01,TC02
 +
|-
 +
! 40
 +
| eth2p-ethip4-ip4base-ip4dhcpclient-func.robot
 +
| dhcp_client.robot TC01..TC05
 +
|-
 +
! 41
 +
| eth2p-ethip4-ip4base-ip4dhcpproxy-func.robot
 +
| dhcp_v4_proxy.robot TC01,TC02
 +
|-
 +
! 42
 +
| eth2p-ethip4-ip4base-ip4ecmp-func.robot
 +
| ipv4_multipath_routing.robot TC01
 +
|-
 +
! 43
 +
| eth2p-ethip4-ip4base-ip4ipfixbase-func.robot
 +
| ipfix_ipv4.robot TC01..TC05
 +
|-
 +
! 44
 +
| eth2p-ethip4-ip4base-ip4ipfixscale-func.robot
 +
| ipfix_ipv4.robot TC06,TC07
 +
|-
 +
! 45
 +
| eth2p-ethip4-ip4base-ip4proxyarp-func.robot
 +
| ipv4_proxy_arp_untagged.robot TC01..TC05
 +
|-
 +
! 46
 +
| eth2p-ethip4-ip4base-ip6dhcpproxy-func.robot
 +
| dhcp_v6_proxy.robot TC01
 +
|-
 +
! 47
 +
| eth2p-ethip4-ip4base-ipolicemarkbase-func.robot
 +
| policer_ipv4_.robot TC01..TC04
 +
|-
 +
! 48
 +
| eth2p-ethip4-ip4base-rpf-func.robot
 +
| ipv4_rpf_untagged.robot TC01..TC02
 +
|-
 +
! 49
 +
| eth2p-ethip4-ip4base-spanrx-func.robot
 +
| span.robot TC01,TC02
 +
|-
 +
! 50
 +
| eth2p-ethip4-ip4basevrf-func.robot
 +
| vrf_tests.robot TC01..TC10
 +
|-
 +
! 51
 +
| eth2p-ethip4gre-ip4base-func.robot
 +
| gre_encapsulation.robot TC01..TC06
 +
|-
 +
! 52
 +
| eth2p-ethip4ipsectnl-ip4base-func.robot
 +
| ipsec_tunnel_mode_ipv4.robot TC01..TC19
 +
|-
 +
! 53
 +
| eth2p-ethip4ipsectpt-ip4base-func.robot
 +
| ipsec_transport_mode_ipv4.robot TC01..TC19
 +
|-
 +
! 54
 +
| eth2p-ethip4ipsectptgpe-ip4base-func.robot
 +
| ipsec_ipv4_lispgpe_ipv4.robot
 +
|-
 +
! 55
 +
| eth2p-ethip4ipsectptlispgpe-ip4base-eth-2vhost-1vm-func.robot
 +
| ipsec_ipv4_lispgpe_ipv4.robot TC05,TC06
 +
|-
 +
! 56
 +
| eth2p-ethip4ipsectptlispgpe-ip4base-func.robot
 +
| ipsec_ipv4_lispgpe_ipv4.robot TC01..TC04
 +
|-
 +
! 57
 +
| eth2p-ethip4ipsectptlispgpe-ip6base-eth-2vhost-1vm-func.robot
 +
| ipsec_ipv6_lispgpe_ipv4.robot TC03,TC04
 +
|-
 +
! 58
 +
| eth2p-ethip4ipsectptlispgpe-ip6base-func.robot
 +
| ipsec_ipv6_lispgpe_ipv4.robot TC01,TC02
 +
|-
 +
! 59
 +
| eth2p-ethip4ipsectptlispgpe-ip6basevrf-func.robot
 +
| ipsec_ipv6_lispgpe_ipv4.robot TC05
 +
|-
 +
! 60
 +
| eth2p-ethip4lisp-ip4base-func.robot
 +
| dataplane_generic.robot TC01
 +
|-
 +
! 61
 +
| eth2p-ethip4lisp-l2bdbasemaclrn-func.robot
 +
| l2_lisp_ipv4.robot TC01
 +
|-
 +
! 62
 +
| eth2p-ethip4lispgpe-ip4base-eth-2vhost-1vm-func.robot
 +
| ipv4_lispgpe_ipv4.robot TC03
 +
|-
 +
! 63
 +
| eth2p-ethip4lispgpe-ip4base-func.robot
 +
| ipv4_lispgpe_ipv4.robot TC01
 +
|-
 +
! 64
 +
| eth2p-ethip4lispgpe-ip4basevrf-eth-2vhost-1vm-func.robot
 +
| ipv4_lispgpe_ipv4.robot TC04
 +
|-
 +
! 65
 +
| eth2p-ethip4lispgpe-ip4basevrf-func.robot
 +
| ipv4_lispgpe_ipv4.robot TC02
 +
|-
 +
! 66
 +
| eth2p-ethip4lispgpe-ip6base-eth-2vhost-1vm-func.robot
 +
| ipv4_lispgpe_ipv6.robot TC03
 +
|-
 +
! 67
 +
| eth2p-ethip4lispgpe-ip6base-func.robot
 +
| ipv4_lispgpe_ipv6.robot TC01
 +
|-
 +
! 68
 +
| eth2p-ethip4lispgpe-ip6basevrf-func.robot
 +
| ipv4_lispgpe_ipv6.robot TC02
 +
|-
 +
! 69
 +
| eth2p-ethip4vxlan-l2bdbasemaclrn--eth-4vhost-2vm-fds-tenant-nets-func.robot
 +
| tenant_network.robot TC01
 +
|-
 +
! 70
 +
| eth2p-ethip4vxlan-l2bdbasemaclrn-eth-2vhost-1vm-func.robot
 +
| vxlan_bd_untagged.robot TC07
 +
|-
 +
! 71
 +
| eth2p-ethip4vxlan-l2bdbasemaclrn-func.robot
 +
| vxlan_bd_untagged.robot TC01
 +
|-
 +
! 72
 +
| eth2p-ethip4vxlan-l2bdbasemaclrn-l2shg-func.robot
 +
| vxlan_bd_untagged.robot TC02,TC03
 +
|-
 +
! 73
 +
| eth2p-ethip4vxlan-l2xcbase-func.robot
 +
| vxlan_xconnect_untagged.robot TC01
 +
|-
 +
! 74
 +
| eth2p-ethip4vxlan-l2xcbase-func.robot
 +
| vxlan_xconnect_untagged.robot TC01
 +
|-
 +
! 75
 +
| eth2p-ethip6-ip6base-copwhlistbase-func.robot
 +
| cop_whitelist_blacklist_IPv6.robot TC01
 +
|-
 +
! 76
 +
| eth2p-ethip6-ip6base-func.robot
 +
| ipv6_untagged.robot TC01..TC08
 +
|-
 +
! 77
 +
| eth2p-ethip6-ip6base-iacldstbase-func.robot
 +
| ipv6_iacl_untagged.robot TC01..TC12
 +
|-
 +
! 78
 +
| eth2p-ethip6-ip6base-ip6ecmp-func.robot
 +
| ipv6_multipath_routing.robot TC01
 +
|-
 +
! 79
 +
| eth2p-ethip6-ip6base-ip6ipfixbase-func.robot
 +
| ipfix_ipv6.robot TC01..TC04
 +
|-
 +
! 80
 +
| eth2p-ethip6-ip6base-ip6ipfixscale-func.robot
 +
| ipfix_ipv6.robot TC05,TC06
 +
|-
 +
! 81
 +
| eth2p-ethip6-ip6base-ip6ra-func.robot
 +
| ipv6_ra.robot TC01..TC04
 +
|-
 +
! 82
 +
| eth2p-ethip6-ip6base-ipolicemarkbase-func.robot
 +
| policer_ipv6_.robot TC01..TC04
 +
|-
 +
! 83
 +
| eth2p-ethip6-ip6base-spanrx-func.robot
 +
| span.robot TC03
 +
|-
 +
! 84
 +
| eth2p-ethip6ipsectnl-ip6base-func.robot
 +
| ipsec_tunnel_mode_ipv6.robot TC01..TC19
 +
|-
 +
! 85
 +
| eth2p-ethip6ipsectpt-ip6base-func.robot
 +
| ipsec_transport_mode_ipv6.robot TC01..TC19
 +
|-
 +
! 86
 +
| eth2p-ethip6ipsectptlispgpe-ip4base-eth-2vhost-1vm-func.robot
 +
| ipsec_ipv4_lispgpe_ipv6.robot TC03,TC04
 +
|-
 +
! 87
 +
| eth2p-ethip6ipsectptlispgpe-ip4base-func.robot
 +
| ipsec_ipv4_lispgpe_ipv6.robot TC01,TC02
 +
|-
 +
! 88
 +
| eth2p-ethip6ipsectptlispgpe-ip6base-eth-2vhost-1vm-func.robot
 +
| ipsec_ipv6_lispgpe_ipv6.robot TC03,TC04
 +
|-
 +
! 89
 +
| eth2p-ethip6ipsectptlispgpe-ip6base-func.robot
 +
| ipsec_ipv6_lispgpe_ipv6.robot TC01,TC02
 +
|-
 +
! 90
 +
| eth2p-ethip6lisp-l2bdbasemaclrn-func.robot
 +
| l2_lisp_ipv6.robot TC01
 +
|-
 +
! 91
 +
| eth2p-ethip6lispgpe-ip4base-func.robot
 +
| ipv6_lispgpe_ipv4.robot TC01
 +
|-
 +
! 92
 +
| eth2p-ethip6lispgpe-ip6base-eth-2vhost-1vm-func.robot
 +
| ipv6_lispgpe_ipv6.robot TC03
 +
|-
 +
! 93
 +
| eth2p-ethip6lispgpe-ip6base-func.robot
 +
| ipv6_lispgpe_ipv6.robot TC01
 +
|-
 +
! 94
 +
| eth2p-ethip6lispgpe-ip6basevrf-eth-2vhost-1vm-func.robot
 +
| ipv6_lispgpe_ipv6.robot TC04
 +
|-
 +
! 95
 +
| eth2p-ethip6lispgpe-ip6basevrf-func.robot
 +
| ipv6_lispgpe_ipv6.robot TC02
 +
|-
 +
! 96
 +
| eth2p-ethip6vxlan-l2bdbasemaclrn-eth-2vhost-1vm-func.robot
 +
| vxlan_bd_untagged.robot TC08
 +
|-
 +
! 97
 +
| eth2p-ethip6vxlan-l2bdbasemaclrn-func.robot
 +
| vxlan_bd_untagged.robot TC04
 +
|-
 +
! 98
 +
| eth2p-ethip6vxlan-l2bdbasemaclrn-l2shg-func.robot
 +
| vxlan_bd_untagged.robot TC05,TC06
 +
|-
 +
! 99
 +
| mgmt-cfg-acl-apihc-apivat-func.robot
 +
| 080_access_control_lists.robot
 +
|-
 +
! 100
 +
| mgmt-cfg-ietfacl-apihc-apivat-func.robot
 +
| 081_ietf_acl_traffic.robot
 +
|-
 +
! 101
 +
| mgmt-cfg-int-apihcnc-func.robot
 +
| 070_netconf.robot
 +
|-
 +
! 102
 +
| mgmt-cfg-int-subint-apihc-apivat-func.robot
 +
| 060_sub_interface.robot
 +
|-
 +
! 103
 +
| mgmt-cfg-intip4-intip6-apihc-apivat-func.robot
 +
| 010_interface_management.robot
 +
|-
 +
! 104
 +
| mgmt-cfg-inttap-apihc-apivat-func.robot
 +
| 040_tap.robot
 +
|-
 +
! 105
 +
| mgmt-cfg-inttap-apivat-func.robot
 +
| tap_interface.robot TC06
 +
|-
 +
! 106
 +
| mgmt-cfg-intvhost-apihc-apivat-func.robot
 +
| 050_interface_vhost_user.robot
 +
|-
 +
! 107
 +
| mgmt-cfg-l2bd-apihc-apivat-func.robot
 +
| 020_bridge_domain.robot
 +
|-
 +
! 108
 +
| mgmt-cfg-l2fib-apihc-apivat-func.robot
 +
| 021_l2_fib.robot
 +
|-
 +
! 109
 +
| mgmt-cfg-lisp-apihc-apivat-func.robot
 +
| 100_lisp.robot
 +
|-
 +
! 110
 +
| mgmt-cfg-lisp-apivat-func.robot
 +
| api.robot TC01..TC05
 +
|-
 +
! 111
 +
| mgmt-cfg-nsh-apihc-apivat-func.robot
 +
| 110_nsh_sfc.robot
 +
|-
 +
! 112
 +
| mgmt-cfg-pbb-apihc-apivat-func.robot
 +
| 090_provider_backbone_bridge.robot
 +
|-
 +
! 113
 +
| mgmt-cfg-snat44-apihc-apivat-func.robot
 +
| 130_nat.robot
 +
|-
 +
! 114
 +
| mgmt-cfg-spanrx-apihc-apivat-func.robot
 +
| 120_port_mirroring.robot
 +
|-
 +
! 115
 +
| mgmt-cfg-vxlan-apihc-apivat-func.robot
 +
| 030_vxlan.robot
 +
|-
 +
! 116
 +
| mgmt-cfg-vxlangpe-apihc-apivat-func.robot
 +
| 031_vxlan_gpe.robot
 +
|-
 +
! 117
 +
| mgmt-notif-apihcnc-func.robot
 +
| 071_notification.robot
 
|-
 
|-
−
| n
+
! 118
−
| to be completed
+
| mgmt-statepersist-apihc-func.robot
−
| to be completed
+
| 900_persistence.robot
 
|}
 
|}

Latest revision as of 13:41, 24 January 2017

CSIT Performance TCs Naming

Background

Now that the number and type of CSIT test cases (TCs) grew significantly, existing naming notation does not work anymore. Firstly, the existing practice does not always follow any strict naming convention. Secondly current TC names do not always clearly capture packet encapsulations, the actual type or content of the tests. Thirdly HW configurations in terms of NICs, ports and their locality are not captured in the name. Bottom line, to address these points within the CSIT project, we agreed to define a new more complete and stricter TC naming convention, and to apply this to all existing and new test cases.

This wiki page captures this change.

CSIT Tests - Naming

Overview

The approach is to use tree naming convention and to encode following testing information into test names:

  1. packet network port configuration
    • physical port type;
    • NIC model;
    • port-NIC locality;
  2. packet encapsulations
  3. VPP packet processing
    • packet forwarding mode
    • packet processing function(s)
  4. packet forwarding path
    • if present, network functions (processes, containers, VMs) and their topology within the computer;
  5. main measured variable, type of test;

Proposed convention is to encode physical ports and NIC on the left (underlay), followed by outer most frame header, then other stacked headers up to the header processed by vSwitch-VPP, then VPP forwarding function, then encap on vhost interface, number of vhost interfaces, number of VMs. If chained VMs present, they get added on the right. Test topology is expected to be symmetric, in other words packets enter and leave SUT through ports specified on the left of the test name. Here some examples to illustrate the convention followed by the complete legend, and tables mapping the new test filenames to old ones.

Convention

CSIT test naming convention for common tested VNF topologies:

  1. Physical port to physical port - a.k.a. NIC-to-NIC, Phy-to-Phy, P2P
    • PortNICConfig-WireEncapsulation-PacketForwardingFunction-PacketProcessingFunction1-...-PacketProcessingFunctionN-TestType
    • 10ge2p1x520-dot1q-l2bdbasemaclrn-ndrdisc.robot => 2 ports of 10GE on Intel x520 NIC, dot1q tagged Ethernet, L2 bridge-domain baseline switching with MAC learning, NDR throughput discovery.
    • 10ge2p1x520-ethip4vxlan-l2bdbasemaclrn-ndrchk.robot => 2 ports of 10GE on Intel x520 NIC, IPv4 VXLAN Ethernet, L2 bridge-domain baseline switching with MAC learning, NDR throughput discovery.
    • 10ge2p1x520-ethip4-ip4base-ndrdisc.robot => 2 ports of 10GE on Intel x520 NIC, IPv4 baseline routed forwarding, NDR throughput discovery.
    • 10ge2p1x520-ethip6-ip6scale200k-ndrdisc.robot => 2 ports of 10GE on Intel x520 NIC, IPv6 scaled up routed forwarding, NDR throughput discovery.
    • 10ge2p1x520-ethip4-ip4base-iacldstbase-ndrdisc.robot => 2 ports of 10GE on Intel x520 NIC, IPv4 baseline routed forwarding, ingress Access Control Lists baseline matching on destination, NDR throughput discovery.
    • 40ge2p1vic1385-ethip4-ip4base-ndrdisc.robot => 2 ports of 40GE on Cisco vic1385 NIC, IPv4 baseline routed forwarding, NDR throughput discovery.
    • eth2p-ethip4-ip4base-func.robot => 2 ports of Ethernet, IPv4 baseline routed forwarding, functional tests.
  2. Physical port to VM (or VM chain) to physical port - a.k.a. NIC2VM2NIC, P2V2P, NIC2VMchain2NIC, P2V2V2P
    • PortNICConfig-WireEncapsulation-PacketForwardingFunction-PacketProcessingFunction1-...-PacketProcessingFunctionN-VirtEncapsulation-VirtPortConfig-VMconfig-TestType
    • 10ge2p1x520-dot1q-l2bdbasemaclrn-eth-2vhost-1vm-ndrdisc.robot => 2 ports of 10GE on Intel x520 NIC, dot1q tagged Ethernet, L2 bridge-domain switching to/from two vhost interfaces and one VM, NDR throughput discovery.
    • 10ge2p1x520-ethip4vxlan-l2bdbasemaclrn-eth-2vhost-1vm-ndrdisc.robot => 2 ports of 10GE on Intel x520 NIC, IPv4 VXLAN Ethernet, L2 bridge-domain switching to/from two vhost interfaces and one VM, NDR throughput discovery.
    • 10ge2p1x520-ethip4vxlan-l2bdbasemaclrn-eth-4vhost-2vm-ndrdisc.robot => 2 ports of 10GE on Intel x520 NIC, IPv4 VXLAN Ethernet, L2 bridge-domain switching to/from four vhost interfaces and two VMs, NDR throughput discovery.
    • eth2p-ethip4vxlan-l2bdbasemaclrn-eth-2vhost-1vm-func.robot => 2 ports of Ethernet, IPv4 VXLAN Ethernet, L2 bridge-domain switching to/from two vhost interfaces and one VM, functional tests.
  3. API CRUD tests - Create (Write), Read (Retrieve), Update (Modify), Delete (Destroy) operations for configuration and operational data
    • ManagementTestKeyword-ManagementOperation-ManagedFunction1-...-ManagedFunctionN-ManagementAPI1-ManagementAPIN-TestType
    • mgmt-cfg-lisp-apivat-func => configuration of LISP with VAT API calls, functional tests.
    • mgmt-cfg-l2bd-apihc-apivat-func => configuration of L2 Bridge-Domain with HoneyComb API and VAT API calls, functional tests.
    • mgmt-oper-int-apihcnc-func => reading status and operational data of interface with HoneyComb NetConf API calls, functional tests.
    • mgmt-cfg-int-tap-apihcnc-func => configuration of tap interfaces with HoneyComb NetConf API calls, functional tests.
    • mgmt-notif-int-subint-apihcnc-func => notifications of interface and sub-interface events with HoneyComb NetConf Notifications, functional tests.

Legend

  1. PortNICConfig - packet network port configuration: physical port type, NIC model, port-NIC locality
    1. <n>ge | eth - port type
      • <n>ge - Gigabit Ethernet, n=10|40|100|200|400.
      • eth - generic Ethernet interface.
    2. <n>p - number of physical Ethernet ports, ports distributed evenly across tested NICs, n=1,2,...
    3. <n><nic_model> - number of NICs and NIC model, n=1,2,...
  2. ManagementTestKeyword - prefix for management API tests, no packet forwarding.
    1. mgmt - management operations test.
  3. WireEncapsulation
    1. eth | dot1q | dot1ad - Ethernet header
      • eth - untagged Ethernet on the physical wire.
      • dot1q - single VLAN tag on physical wire.
      • dot1ad - double VLAN tag on physical wire.
      • -- - OR separator, if present multiple encapsulations present on interfaces.
    2. ip4 | ip6 | ip6e - outer IP header
      • ip4 - IPv4 header.
      • ip6 - IPv6 header.
      • ip6e - IPv6 extended header.
      • ip6ip4 - IPv4-over-IPv6 softwire header.
      • -- - OR separator, if present multiple encapsulations present on interfaces.
    3. ipsectrt | ipsectnl - IPSec crypto
      • ipsectrt - IPSec transport mode.
      • ipsectnl - IPSec tunnel mode.
    4. vxlan | lisp | gre | gpe - tunnel header
      • vxlan - VXLAN tunnel header.
      • lisp - LISP tunnel header.
      • gre - GRE tunnel header.
      • gpe - GPE tunnel header.
  4. ManagementOperation
    1. cfg | oper | notif | statepersist - management operation type
      • cfg - configuration writes, modifies, deletes and read verification.
      • oper - operational state reads.
      • notif - notifications generation on events.
      • statepersist - configuration and operation state persistence.
  5. PacketForwardingFunction
    1. l2bd - VPP L2 bridge-domain
      1. base | scale
        • base - baseline test with one L2 MAC flow received per interface.
        • scale<n> - scale test with many L2 MAC flows received per interface, <n> - number of 48-bit MAC address entries present in L2 MAC lookup table.
      2. maclrn | macstc
        • maclrn - learning L2 MAC addresses or MAC flows.
        • macstc - static L2 MAC addresses or MAC flows, programmed before the test.
    2. l2xc - VPP L2 point-to-point crossconnect
      1. base | scale
        • base - baseline test with one L2 cross-connect configured.
        • scale<n> - scale test with <n> L2 cross-connects configured.
    3. ip4 - VPP IPv4 routed forwarding
      1. base | scale
        • base - baseline test with one IPv4 flow received per interface.
        • scale<n> - scale tests with many IPv4 flows received per interface, <n> - number of /32 IPv4 address entries present in IPv4 lookup table.
    4. ip6 - VPP IPv6 routed forwarding
      1. base | scale
        • base - baseline test with one IPv6 flow received per interface.
        • scale<n> - scale tests with many IPv6 flows received per interface, <n> - number of /128 IPv6 address entries present in IPv6 lookup table.
  6. ManagedFunction
    1. int - interface.
    2. subint - sub-interface.
    3. inttap - tap interface.
    4. intip4 - IPv4 interface.
    5. intip6 - IPv6 interface.
    6. l2bd - L2 Bridge-Domain.
    7. acl - Access Control List.
    8. ietfacl - IETF-style Access Control List.
    9. spanrx - Switched Port Analyzer (SPAN) port mirroring of Rx'ed packets.
    10. l2fib - L2 Forwarding Information Base a.k.a. MAC lookup and forwarding table.
    11. snat44 - static Network Address Translation (NAT) for IPv4 to IPv4.
    12. nsh - Network Service Header encapsulation functions.
    13. vxlangpe - VXLAN GPE encapsulation functions.
    14. pbb - PBB Provider Backbone Bridge encapsulation functions.
  7. PacketProcessingFunction
    1. l2shg - L2 Split Horizon Group.
    2. vlantrans - VLAN-ID tag translation
      1. 11 | 12 | 21 | 22
        • 11 - one-to-one VLAN-ID tag translation, dot1q to dot1q.
        • 12 - one-to-two VLAN-ID tag translation, dot1q to dot1ad.
        • 21 - two-to-one VLAN-ID tag translation, dot1ad to dot1q.
        • 22 - two-to-two VLAN-ID tag translation, dot1ad to dot1ad.
    3. iacldstbase - ingress Access Control Lists baseline matching on destination.
    4. ipolicemarkbase - ingress policer and marker baseline.
    5. copwhtlistbase - ingress IP Cop security whitelist filter entries, /24 for IPv4, /64 for IPv6.
    6. copblklistbase - ingress IP Cop security blacklist filter entries, /24 for IPv4, /64 for IPv6
    7. rpf - IP Reverse Path Forwarding check.
    8. swirelw46 - softwire IPv4-over-IPv6 tunnel concentrator, LightWeight-IPv4-over-IPv6, LW46.
    9. swiremape - softwire IPv4-over-IPv6 tunnel concentrator, MAP-E.
    10. swiremapt - softwire IPv4-over-IPv6 tunnel concentrator, MAP-T.
    11. ip4ipfix - IPv4 IPFIX flow statistics
      1. base | scale
        • base - baseline tests with one IPv4 flow received per interface.
        • scale - scale tests with many IPv4 flows received per interface.
    12. ip6ipfix - IPv6 IPFIX flow statistics
      1. base | scale
        • base - baseline tests with one IPv6 flow received per interface.
        • scale - scale tests with many IPv6 flows received per interface.
    13. spanrx - Switched Port Analyzer (SPAN) port mirroring of Rx'ed packets, receive direction.
    14. spantx - Switched Port Analyzer (SPAN) port mirroring of Tx'ed packets, transmit direction.
    15. ip4dhcpclient - IPv4 DHCP client.
    16. ip4dhcpcproxy - IPv4 DHCP proxy.
  8. ManagementAPI
    1. apihc - HoneyComb API.
    2. apihcnc - HoneyComb NetConf API.
    3. apivat - "VPP API Test" (VAT) API.
  9. VirtEncapsulation
    1. eth | dot1q | dot1ad
      • eth - untagged Ethernet on virtual interface.
      • dot1q - single VLAN tag on virtual interface.
      • dot1ad - double VLAN tag on virtual interface.
  10. VirtPortConfig
    1. <n>vhost - vhost virtual interface, n=1,2, ...
    2. <n>tap - tap virtual interface, n=1,2, ...
  11. VMconfig
    1. <n>vm - Virtual Machines, n=1,2, ...
  12. TestType
    1. ndrdisc | ndrchk | pdrdisc | pdrchk | func
      • ndrdisc - NDR throughput discovery using RFC2544 binary search.
      • ndrchk - NDR throughput check against a reference NDR throughput.
      • pdrdisc - PDR throughput discovery using RFC2544 binary search.
      • pdrchk - PDR throughput check against a reference NDR throughput.
      • func - functional tests.

Examples

CSIT Perf - Suites and Test Cases

Below examples of CSIT Performance Suite Names (filename.robot) and included test cases.

  1. suite: 10ge2p1x520-ethip4-ip4base-ndrdisc.robot
    1. test cases:
      1. tc01-64B-2t1c-ethip4-ip4base-ndrdisc
      2. tc02-IMIX-1t1c-ethip4-ip4base-ndrdisc
      3. tc03-1518B-1t1c-ethip4-ip4base-ndrdisc
  2. suite: 10ge2p1x710-ethip4-ip4base-ndrdisc.robot
    1. test cases:
      1. tc01-64B-2t1c-ethip4-ip4base-ndrdisc
      2. tc02-IMIX-1t1c-ethip4-ip4base-ndrdisc
      3. tc03-1518B-1t1c-ethip4-ip4base-ndrdisc
  3. suite: 40ge2p1xl710-ethip4-ip4base-ndrdisc.robot
    1. test cases:
      1. tc01-64B-2t1c-ethip4-ip4base-ndrdisc
      2. tc02-IMIX-1t1c-ethip4-ip4base-ndrdisc
      3. tc03-1518B-1t1c-ethip4-ip4base-ndrdisc

CSIT Perf - Trending Graphs

Below examples of CSIT Perf Trending Graphs and included test case result plots.

  1. graph: 64B-1t1c-ethip4-ip4*-ndrdisc
    1. test case result plots:
      1. 10ge2p1x520-ethip4-ip4base
      2. 10ge2p1x710-ethip4-ip4base
      3. 40ge2p1xl710-ethip4-ip4base
      4. 10ge2p1x520-ethip4-ip4scale20k
      5. 10ge2p1x520-ethip4-ip4scale200k
      6. 10ge2p1x520-ethip4-ip4scale2m
      7. 10ge2p1x520-ethip4-ip4base-copwhtlistbase
      8. 10ge2p1x520-ethip4-ip4base-iacldstbase
      9. 10ge2p1x520-ethip4-ip4base-ipolicemarkbase
  2. graph: 64B-1t1c-ethip6-ip6*-ndrdisc
    1. test case result plots:
      1. 10ge2p1x520-ethip6-ip6base
      2. 10ge2p1x710-ethip6-ip6base
      3. 40ge2p1xl710-ethip6-ip6base
      4. 10ge2p1x520-ethip6-ip6scale20k
      5. 10ge2p1x520-ethip6-ip6scale200k
      6. 10ge2p1x520-ethip6-ip6scale2m
      7. 10ge2p1x520-ethip6-ip6base-copwhtlistbase
      8. 10ge2p1x520-ethip6-ip6base-iacldstbase
      9. 10ge2p1x520-ethip6-ip6base-ipolicemarkbase

CSIT Performance Tests

Performance tests executed in FD.io CSIT CPL (Continuous Performance Lab) physical testbeds. Naming change code in FD.io CSIT project by patch #3856.

Throughput NDR Discovery

Sorted alphabetically by new name.

# New Test Filename Old Test Filename
1 10ge2p1x520-dot1ad-l2xcbase-ndrdisc.robot Long_Xconnect_Dot1ad_Intel-X520-DA2.robot
2 10ge2p1x520-dot1q-l2xcbase-ndrdisc.robot Long_Xconnect_Dot1q_Intel-X520-DA2.robot
3 10ge2p1x520-eth-l2bdbasemaclrn-eth-2vhost-1vm-ndrdisc.robot Long_Bridge_Domain_Vhost_Intel-X520-DA2.robot
4 10ge2p1x520-eth-l2bdbasemaclrn-ndrdisc.robot Bridge_Domain_Intel-X520-DA2.robot
5 10ge2p1x520-eth-l2xcbase-eth-2vhost-1vm-ndrdisc.robot Long_Xconnect_Vhost_Intel-X520-DA2.robot
6 10ge2p1x520-eth-l2xcbase-ndrdisc.robot Long_Xconnect_Intel-X520-DA2.robot
7 10ge2p1x520-ethip4-ip4base-copwhtlistbase-ndrdisc.robot Long_IPv4_Cop_Intel-X520-DA2.robot
8 10ge2p1x520-ethip4-ip4base-eth-2vhost-1vm-ndrdisc.robot Long_IPv4_Vhost_Intel-X520-DA2.robot
9 10ge2p1x520-ethip4-ip4base-iacldstbase-ndrdisc.robot Long_IPv4_iAcl_Intel-X520-DA2.robot
10 10ge2p1x520-ethip4-ip4base-ndrdisc.robot Long_IPv4_Intel-X520-DA2.robot
11 10ge2p1x520-ethip4-ip4base-ipolicemarkbase-ndrdisc.robot Long_IPv4_Policer_Intel-X520-DA2.robot
12 10ge2p1x520-ethip4-ip4scale200k-ndrdisc.robot Long_IPv4_Fib_200k_Intel-X520-DA2.robot
13 10ge2p1x520-ethip4-ip4scale20k-ndrdisc.robot Long_IPv4_Fib_20k_Intel-X520-DA2.robot
14 10ge2p1x520-ethip4-ip4scale2m-ndrdisc.robot Long_IPv4_Fib_2M_Intel-X520-DA2.robot
15 10ge2p1x520-ethip4lispip4-ip4base-ndrdisc.robot Long_Lisp_Intel-X520-DA2.robot
16 10ge2p1x520-ethip4lispip6-ip4base-ndrdisc.robot Long_Lisp_Intel-X520-DA2.robot
17 10ge2p1x520-ethip4vxlan-l2xcbase-ndrdisc.robot Long_Xconnect_Vxlan_Ipv4_Intel-X520-DA2.robot
18 10ge2p1x520-ethip6-ip6base-copwhtlistbase-ndrdisc.robot Long_IPv6_Cop_Intel-X520-DA2.robot
19 10ge2p1x520-ethip6-ip6base-iacldstbase-ndrdisc.robot Long_IPv6_iAcl_Intel-X520-DA2.robot
20 10ge2p1x520-ethip6-ip6base-ndrdisc.robot Long_IPv6_Intel-X520-DA2.robot
21 10ge2p1x520-ethip6-ip6scale200k-ndrdisc.robot Long_IPv6_Fib_200k_Intel-X520-DA2.robot
22 10ge2p1x520-ethip6-ip6scale20k-ndrdisc.robot Long_IPv6_Fib_20k_Intel-X520-DA2.robot
23 10ge2p1x520-ethip6-ip6scale2m-ndrdisc.robot Long_IPv6_Fib_2M_Intel-X520-DA2.robot
24 10ge2p1x520-ethip6lispip4-ip6base-ndrdisc.robot Long_Lisp_Intel-X520-DA2.robot
25 10ge2p1x520-ethip6lispip6-ip6base-ndrdisc.robot Long_Lisp_Intel-X520-DA2.robot
26 10ge2p1x710-eth-l2bdbasemaclrn-ndrdisc.robot Bridge_Domain_Intel-X710-DA2.robot
27 40ge2p1xl710-eth-l2bdbasemaclrn-ndrdisc.robot Bridge_Domain_Intel-XL710.robot
28 40ge2p1xl710-eth-l2xcbase-ndrdisc.robot Long_Xconnect_Intel-XL710.robot
29 40ge2p1xl710-ethip4-ip4base-ndrdisc.robot Long_IPv4_Intel-XL710.robot
30 40ge2p1xl710-ethip6-ip6base-ndrdisc.robot Long_IPv6_Intel-XL710.robot

Throughput NDR Check

# New Test Filename Old Test Filename
1 10ge2p1x520-dot1ad-l2xcbase-ndrchk.robot Short_Xconnect_Dot1ad_Intel-X520-DA2.robot
2 10ge2p1x520-dot1q-l2xcbase-ndrchk.robot Short_Xconnect_Dot1q_Intel-X520-DA2.robot
3 10ge2p1x520-eth-l2bdbasemaclrn-ndrchk.robot Short_Bridge_Domain_Intel-X520-DA2.robot
4 10ge2p1x520-eth-l2xcbase-ndrchk.robot Short_Xconnect_Intel-X520-DA2.robot
5 10ge2p1x520-ethip4-ip4base-copwhtlistbase-ndrchk.robot Short_IPv4_Cop_Intel-X520-DA2.robot
6 10ge2p1x520-ethip4-ip4base-iacldstbase-ndrchk.robot Short_IPv4_iAcl_Intel-X520-DA2.robot
7 10ge2p1x520-ethip4-ip4base-ndrchk.robot Short_IPv4_Intel-X520-DA2.robot
8 10ge2p1x520-ethip4-ip4base-ipolicemarkbase-ndrchk.robot Short_IPv4_Policer_Intel-X520-DA2.robot
9 10ge2p1x520-ethip4-ip4scale200k-ndrchk.robot Short_IPv4_Fib_200k_Intel-X520-DA2.robot
10 10ge2p1x520-ethip4-ip4scale20k-ndrchk.robot Short_IPv4_Fib_20k_Intel-X520-DA2.robot
11 10ge2p1x520-ethip4-ip4scale2m-ndrchk.robot Short_IPv4_Fib_2M_Intel-X520-DA2.robot
12 10ge2p1x520-ethip4lispip4-ip4base-ndrchk.robot Short_Lisp_Intel-X520-DA2.robot
13 10ge2p1x520-ethip4lispip6-ip4base-ndrchk.robot Short_Lisp_Intel-X520-DA2.robot
14 10ge2p1x520-ethip4vxlan-l2xcbase-ndrchk.robot Short_Xconnect_Vxlan_Ipv4_Intel-X520-DA2.robot
15 10ge2p1x520-ethip6-ip6base-copwhtlistbase-ndrchk.robot Short_IPv6_Cop_Intel-X520-DA2.robot
16 10ge2p1x520-ethip6-ip6base-iacldstbase-ndrchk.robot Short_IPv6_iAcl_Intel-X520-DA2.robot
17 10ge2p1x520-ethip6-ip6base-ndrchk.robot Short_IPv6_Intel-X520-DA2.robot
18 10ge2p1x520-ethip6-ip6scale200k-ndrchk.robot Short_IPv6_Fib_200k_Intel-X520-DA2.robot
19 10ge2p1x520-ethip6-ip6scale20k-ndrchk.robot Short_IPv6_Fib_20k_Intel-X520-DA2.robot
20 10ge2p1x520-ethip6-ip6scale2m-ndrchk.robot Short_IPv6_Fib_2M_Intel-X520-DA2.robot
21 10ge2p1x520-ethip6lispip4-ip6base-ndrchk.robot Short_Lisp_Intel-X520-DA2.robot
22 10ge2p1x520-ethip6lispip6-ip6base-ndrchk.robot Short_Lisp_Intel-X520-DA2.robot

Performance Trending Graphs

One of the CSIT goals is to auto-generate VPP performance trending graphs for tested use cases and configurations. Trending graphs are auto-generated for Jenkins csit-perf* jobs using Jenkins plugin <add link> and for CSIT release reports using separate python scripts <add link>. For better readibility VPP performance results are marked with RF tags and grouped into separate graphs as follows:

  1. L2BD, L2XC
    • tags: L2BDMACSTAT or L2BDMACLRN or L2XCFWD
  2. IPv4 base, scale, feature
    • tags: IP4FWD
  3. IPv6 base, scale, feature
    • tags: IP6FWD
  4. Overlay tunnels - LISP, VXLAN, GPE, GRE
    • tags: VXLAN or LISP or LISPGPE or VXLANGPE or GRE
  5. Crypto in software: IP4FWD, IP6FWD
    • tags: IPSECSW and (IPSECTRAN or IPSECTUN)
  6. Crypto in hardware: IP4FWD, IP6FWD
    • tags: IPSECHW and (IPSECTRAN or IPSECTUN)
  7. Overlay tunnels with crypto in software
    • tags: (VXLAN or LISP or LISPGPE or VXLANGPE or GRE) and IPSECSW and (IPSECTRAN or IPSECTUN)
  8. Overlay tunnels with crypto in hardware
    • tags: (VXLAN or LISP or LISPGPE or VXLANGPE or GRE) and IPSEHW and (IPSECTRAN or IPSECTUN)
  9. vhost-user
    • tags: VHOST and (ETH or DOT1Q or VXLAN)

For each of the above, separate graph series is auto-generated for:

  1. Throughput type
    1. NDR
      • tags: NDRDISC
    2. PDR
      • tags: PDRDISC
  2. Worker thread and core configuration
    1. 1t1c - 1 worker-thread, 1 core
      • tags: 1T1C
    2. 2t2c - 2 worker-threads, 2 cores
      • tags: 2T2C
    3. 4t4c - 4 worker-threads, 4 cores
      • tags: 2T2C

Resulting total number of auto-generated graphs: 9*5 = 45 graphs.

Test RF Tagging for Trending Graphs

In order to group performance test results into auto-generated trending graphs, CSIT is using Robot Framework (RF) tags. CSIT RF tags are defined and described in `/docs/tag_documentation.rst'. RF tag changes related to test naming change are captured in patch #3856.

  1. Functional and performance tags
    1. FUNCTEST - All functional test cases.
    2. PERFTEST - All performance test cases.
  2. Performance testing tags
    1. PDRDISC - Partial Drop Rate evaluation of single run result, with non-zero packet loss tolerance (LT) expressed in percentage of packets transmitted
    2. NDRDISC - Non Drop Rate evaluation of results. Loss acceptance of dropped packets is set to zero lost packets
    3. NDRCHK - Performance tests where TG verifies DUTs' throughput at ref-NDR (reference Non Drop Rate) with zero packet loss tolerance
    4. NDRPDRDISC - Find performance of DUT based on RFC2544 with linear / binary / combined search. (Previous LONG tests.)
  3. Test type tags
    1. BASE - Baseline test cases, no encapsulation, no feature(s) configured in tests.
    2. SCALE - Scale test cases.
    3. ENCAP - Test cases where encapsulation is used. Use also encapsulation tag(s).
    4. FEATURE - At least one feature is configured in test cases. Use also feature tag(s).
  4. Scaling tags
    1. FIB_20K - 2x10,000 entries in single fib table
    2. FIB_200K - 2x100,000 entries in single fib table
    3. FIB_2M - 2x1,000,000 entries in single fib table
  5. Forwarding mode tags
    1. L2BDMACSTAT - VPP L2 bridge-domain, L2 MAC static.
    2. L2BDMACLRN - VPP L2 bridge-domain, L2 MAC learning.
    3. L2XCFWD - VPP L2 point-to-point cross-connect.
    4. IP4FWD - VPP IPv4 routed forwarding.
    5. IP6FWD - VPP IPv6 routed forwarding.
  6. Underlay tags
    1. IP4UNRLAY - IPv4 underlay.
    2. IP6UNRLAY - IPv6 underlay.
    3. MPLSUNRLAY - MPLS underlay.
  7. Overlay tags
    1. L2OVRLAY - L2 overlay.
    2. IP4OVRLAY - IPv4 overlay (IPv4 payload).
    3. IP6OVRLAY - IPv6 overlay (IPv6 payload).
  8. Encapsulation tags
    1. ETH - All test cases with base Ethernet (no encapsulation).
    2. DOT1Q - All test cases with dot1q.
    3. DOT1AD - All test cases with dot1ad.
    4. LISP - All test cases with LISP.
    5. LISPGPE - All test cases with LISP-GPE.
    6. VXLAN - All test cases with Vxlan.
    7. VXLANGPE - All test cases with VXLAN-GPE.
    8. GRE - All test cases with GRE.
  9. Interface tags
    1. PHY - All test cases which use physical interface(s).
    2. VHOST - All test cases which uses VHOST.
    3. TUNTAP - All test cases which uses TUN and TAP.
    4. AFPKT - All test cases which uses AFPKT.
    5. NETMAP - All test cases which uses Netmap.
  10. Feature tags
    1. IACLDST - iACL destination.
    2. COPWHLIST - COP whitelist.
  11. Client-workload tags
    1. VM - All test cases which use at least one virtual machine.
    2. LXC - All test cases which use Linux container.
    3. APP - All test cases with specific APP use.
  12. Multi-threading tags
    1. STHREAD - All test cases using single poll mode thread.
    2. MTHREAD - All test cases using more then one poll mode driver thread.
    3. 1NUMA - All test cases with packet processing on single socket.
    4. 2NUMA - All test cases with packet processing on two sockets.
    5. SMT - All test cases with symmetric Multi-Threading (HyperThreading) enabled.
    6. NOSMT - All test cases with symmetric Multi-Threading (HyperThreading) disabled.
    7. 1T1C - 1 worker thread pinned to 1 dedicated physical core. 1 receive queue per interface. Main thread pinned to core 0.
    8. 2T2C - 2 worker threads pinned to 2 dedicated physical cores. 1 receive queue per interface. Main thread pinned to core 0.
    9. 4T4C - 4 worker threads pinned to 4 dedicated physical cores. 2 receive queues per interface. Main thread pinned to core 0.
    10. 6T6C - 6 worker threads pinned to 6 dedicated physical cores. 3 receive queues per interface. Main thread pinned to core 0.
    11. 8T8C - 8 worker threads pinned to 8 dedicated physical cores. 4 receive queues per interface. Main thread pinned to core 0.

CSIT Functional Tests

Functional tests executed in FD.io CSIT VIRL testbeds. Functional tests naming change coded in FD.io CSIT project by patch #4753.

Sorted alphabetically by new name.

# New Test Filename Old Test Filename and TC refs
1 eth2p-dot1ad--dot1q-l2bdbasemaclrn-vlantrans21-func.robot vlan_tag_translate_l2_bridge_domain_ipv4.robot TC07,TC08
2 eth2p-dot1ad--dot1q-l2bdbasemaclrn-vlantrans21-func.robot vlan_tag_translate_l2_bridge_domain_ipv6.robot TC07,TC08
3 eth2p-dot1ad--dot1q-l2xcbase-vlantrans21-func.robot vlan_tag_translate_l2_xconnect_ipv4.robot TC07,TC08
4 eth2p-dot1ad--dot1q-l2xcbase-vlantrans21-func.robot vlan_tag_translate_l2_xconnect_ipv6.robot TC07,TC08
5 eth2p-dot1ad-l2bdbasemaclrn-vlantrans22-func.robot vlan_tag_translate_l2_bridge_domain_ipv4.robot TC09..TC12
6 eth2p-dot1ad-l2bdbasemaclrn-vlantrans22-func.robot vlan_tag_translate_l2_bridge_domain_ipv6.robot TC09..TC12
7 eth2p-dot1ad-l2xcbase-func.robot qinq_l2_xconnect.robot TC01
8 eth2p-dot1ad-l2xcbase-vlantrans22-func.robot vlan_tag_translate_l2_xconnect_ipv4.robot TC09..TC12
9 eth2p-dot1ad-l2xcbase-vlantrans22-func.robot vlan_tag_translate_l2_xconnect_ipv6.robot TC09..TC12
10 eth2p-dot1q--dot1ad-l2bdbasemaclrn-vlantrans12-func.robot vlan_tag_translate_l2_bridge_domain_ipv4.robot TC03..TC06
11 eth2p-dot1q--dot1ad-l2bdbasemaclrn-vlantrans12-func.robot vlan_tag_translate_l2_bridge_domain_ipv6.robot TC03..TC06
12 eth2p-dot1q--dot1ad-l2xcbase-vlantrans12-func.robot vlan_tag_translate_l2_xconnect_ipv4.robot TC03..TC06
13 eth2p-dot1q--dot1ad-l2xcbase-vlantrans12-func.robot vlan_tag_translate_l2_xconnect_ipv6.robot TC03..TC06
14 eth2p-dot1q-l2bdbasemaclrn-eth-4vhost-2vm-fds-provider-nets-func.robot provider_network.robot TC01
15 eth2p-dot1q-l2bdbasemaclrn-vlantrans11-ipfunc.robot vlan_tag_translate_l2_bridge_domain_ipv4.robot TC01,TC02
16 eth2p-dot1q-l2bdbasemaclrn-vlantrans11-ipfunc.robot vlan_tag_translate_l2_bridge_domain_ipv6.robot TC01,TC02
17 eth2p-dot1q-l2xcbase-vlantrans11-func.robot vlan_tag_translate_l2_xconnect_ipv4.robot TC01,TC02
18 eth2p-dot1q-l2xcbase-vlantrans11-func.robot vlan_tag_translate_l2_xconnect_ipv6.robot TC01,TC02
19 eth2p-dot1qip4vxlan-l2bdbasemaclrn-func.robot vxlan_bd_dot1q.robot TC01
21 eth2p-eth-l2bdbasemaclrn-eth-2tap-func.robot tap_interface.robot TC01
22 eth2p-eth-l2bdbasemaclrn-eth-2vhost-1vm-func.robot bridge_domain_untagged.robot TC06,TC07
23 eth2p-eth-l2bdbasemaclrn-func.robot bridge_domain_untagged.robot TC01..TC03
24 eth2p-eth-l2bdbasemaclrn-l2shg-eth-2tap-func.robot tap_interface.robot TC04,TC05
25 eth2p-eth-l2bdbasemaclrn-l2shg-func.robot bridge_domain_untagged.robot TC05
26 eth2p-eth-l2bdbasemacstc-eth-2vhost-1vm-func.robot bridge_domain_untagged.robot TC08,TC09
27 eth2p-eth-l2bdbasemacstc-func.robot bridge_domain_untagged.robot TC04
28 eth2p-eth-l2xcbase-eth-2vhost-1vm-func.robot l2_xconnect_untagged.robot TC03,TC04
29 eth2p-eth-l2xcbase-func.robot l2_xconnect_untagged.robot TC01,TC02
30 eth2p-eth-l2xcbase-iacldstbase-func.robot l2_iacl_untagged.robot TC01..TC04
31 eth2p-ethip4--ethip6ip4-ip4base--ip6base-swirelw46-func.robot lightweight_4over6.robot TC01..TC04
32 eth2p-ethip4--ethip6ip4-ip4base--ip6base-swiremape-func.robot map_e.robot TC01..TC09
33 eth2p-ethip4--ethip6ip4-ip4base--ip6base-swiremapt-func.robot map_t.robot TC01
34 eth2p-ethip4-ip4base-copblklistbase-func.robot cop_whitelist_blacklist.robot TC02
35 eth2p-ethip4-ip4base-copwhlistbase-func.robot cop_whitelist_blacklist.robot TC01
36 eth2p-ethip4-ip4base-eth-1tap-func.robot tap_interface.robot TC02,TC03
37 eth2p-ethip4-ip4base-func.robot ipv4_untagged.robot TC01..TC08
38 eth2p-ethip4-ip4base-iacldstbase-func.robot ipv4_iacl_untagged.robot TC01..TC11
39 eth2p-ethip4-ip4base-ip4arp-func.robot ipv4_arp_untagged.robot TC01,TC02
40 eth2p-ethip4-ip4base-ip4dhcpclient-func.robot dhcp_client.robot TC01..TC05
41 eth2p-ethip4-ip4base-ip4dhcpproxy-func.robot dhcp_v4_proxy.robot TC01,TC02
42 eth2p-ethip4-ip4base-ip4ecmp-func.robot ipv4_multipath_routing.robot TC01
43 eth2p-ethip4-ip4base-ip4ipfixbase-func.robot ipfix_ipv4.robot TC01..TC05
44 eth2p-ethip4-ip4base-ip4ipfixscale-func.robot ipfix_ipv4.robot TC06,TC07
45 eth2p-ethip4-ip4base-ip4proxyarp-func.robot ipv4_proxy_arp_untagged.robot TC01..TC05
46 eth2p-ethip4-ip4base-ip6dhcpproxy-func.robot dhcp_v6_proxy.robot TC01
47 eth2p-ethip4-ip4base-ipolicemarkbase-func.robot policer_ipv4_.robot TC01..TC04
48 eth2p-ethip4-ip4base-rpf-func.robot ipv4_rpf_untagged.robot TC01..TC02
49 eth2p-ethip4-ip4base-spanrx-func.robot span.robot TC01,TC02
50 eth2p-ethip4-ip4basevrf-func.robot vrf_tests.robot TC01..TC10
51 eth2p-ethip4gre-ip4base-func.robot gre_encapsulation.robot TC01..TC06
52 eth2p-ethip4ipsectnl-ip4base-func.robot ipsec_tunnel_mode_ipv4.robot TC01..TC19
53 eth2p-ethip4ipsectpt-ip4base-func.robot ipsec_transport_mode_ipv4.robot TC01..TC19
54 eth2p-ethip4ipsectptgpe-ip4base-func.robot ipsec_ipv4_lispgpe_ipv4.robot
55 eth2p-ethip4ipsectptlispgpe-ip4base-eth-2vhost-1vm-func.robot ipsec_ipv4_lispgpe_ipv4.robot TC05,TC06
56 eth2p-ethip4ipsectptlispgpe-ip4base-func.robot ipsec_ipv4_lispgpe_ipv4.robot TC01..TC04
57 eth2p-ethip4ipsectptlispgpe-ip6base-eth-2vhost-1vm-func.robot ipsec_ipv6_lispgpe_ipv4.robot TC03,TC04
58 eth2p-ethip4ipsectptlispgpe-ip6base-func.robot ipsec_ipv6_lispgpe_ipv4.robot TC01,TC02
59 eth2p-ethip4ipsectptlispgpe-ip6basevrf-func.robot ipsec_ipv6_lispgpe_ipv4.robot TC05
60 eth2p-ethip4lisp-ip4base-func.robot dataplane_generic.robot TC01
61 eth2p-ethip4lisp-l2bdbasemaclrn-func.robot l2_lisp_ipv4.robot TC01
62 eth2p-ethip4lispgpe-ip4base-eth-2vhost-1vm-func.robot ipv4_lispgpe_ipv4.robot TC03
63 eth2p-ethip4lispgpe-ip4base-func.robot ipv4_lispgpe_ipv4.robot TC01
64 eth2p-ethip4lispgpe-ip4basevrf-eth-2vhost-1vm-func.robot ipv4_lispgpe_ipv4.robot TC04
65 eth2p-ethip4lispgpe-ip4basevrf-func.robot ipv4_lispgpe_ipv4.robot TC02
66 eth2p-ethip4lispgpe-ip6base-eth-2vhost-1vm-func.robot ipv4_lispgpe_ipv6.robot TC03
67 eth2p-ethip4lispgpe-ip6base-func.robot ipv4_lispgpe_ipv6.robot TC01
68 eth2p-ethip4lispgpe-ip6basevrf-func.robot ipv4_lispgpe_ipv6.robot TC02
69 eth2p-ethip4vxlan-l2bdbasemaclrn--eth-4vhost-2vm-fds-tenant-nets-func.robot tenant_network.robot TC01
70 eth2p-ethip4vxlan-l2bdbasemaclrn-eth-2vhost-1vm-func.robot vxlan_bd_untagged.robot TC07
71 eth2p-ethip4vxlan-l2bdbasemaclrn-func.robot vxlan_bd_untagged.robot TC01
72 eth2p-ethip4vxlan-l2bdbasemaclrn-l2shg-func.robot vxlan_bd_untagged.robot TC02,TC03
73 eth2p-ethip4vxlan-l2xcbase-func.robot vxlan_xconnect_untagged.robot TC01
74 eth2p-ethip4vxlan-l2xcbase-func.robot vxlan_xconnect_untagged.robot TC01
75 eth2p-ethip6-ip6base-copwhlistbase-func.robot cop_whitelist_blacklist_IPv6.robot TC01
76 eth2p-ethip6-ip6base-func.robot ipv6_untagged.robot TC01..TC08
77 eth2p-ethip6-ip6base-iacldstbase-func.robot ipv6_iacl_untagged.robot TC01..TC12
78 eth2p-ethip6-ip6base-ip6ecmp-func.robot ipv6_multipath_routing.robot TC01
79 eth2p-ethip6-ip6base-ip6ipfixbase-func.robot ipfix_ipv6.robot TC01..TC04
80 eth2p-ethip6-ip6base-ip6ipfixscale-func.robot ipfix_ipv6.robot TC05,TC06
81 eth2p-ethip6-ip6base-ip6ra-func.robot ipv6_ra.robot TC01..TC04
82 eth2p-ethip6-ip6base-ipolicemarkbase-func.robot policer_ipv6_.robot TC01..TC04
83 eth2p-ethip6-ip6base-spanrx-func.robot span.robot TC03
84 eth2p-ethip6ipsectnl-ip6base-func.robot ipsec_tunnel_mode_ipv6.robot TC01..TC19
85 eth2p-ethip6ipsectpt-ip6base-func.robot ipsec_transport_mode_ipv6.robot TC01..TC19
86 eth2p-ethip6ipsectptlispgpe-ip4base-eth-2vhost-1vm-func.robot ipsec_ipv4_lispgpe_ipv6.robot TC03,TC04
87 eth2p-ethip6ipsectptlispgpe-ip4base-func.robot ipsec_ipv4_lispgpe_ipv6.robot TC01,TC02
88 eth2p-ethip6ipsectptlispgpe-ip6base-eth-2vhost-1vm-func.robot ipsec_ipv6_lispgpe_ipv6.robot TC03,TC04
89 eth2p-ethip6ipsectptlispgpe-ip6base-func.robot ipsec_ipv6_lispgpe_ipv6.robot TC01,TC02
90 eth2p-ethip6lisp-l2bdbasemaclrn-func.robot l2_lisp_ipv6.robot TC01
91 eth2p-ethip6lispgpe-ip4base-func.robot ipv6_lispgpe_ipv4.robot TC01
92 eth2p-ethip6lispgpe-ip6base-eth-2vhost-1vm-func.robot ipv6_lispgpe_ipv6.robot TC03
93 eth2p-ethip6lispgpe-ip6base-func.robot ipv6_lispgpe_ipv6.robot TC01
94 eth2p-ethip6lispgpe-ip6basevrf-eth-2vhost-1vm-func.robot ipv6_lispgpe_ipv6.robot TC04
95 eth2p-ethip6lispgpe-ip6basevrf-func.robot ipv6_lispgpe_ipv6.robot TC02
96 eth2p-ethip6vxlan-l2bdbasemaclrn-eth-2vhost-1vm-func.robot vxlan_bd_untagged.robot TC08
97 eth2p-ethip6vxlan-l2bdbasemaclrn-func.robot vxlan_bd_untagged.robot TC04
98 eth2p-ethip6vxlan-l2bdbasemaclrn-l2shg-func.robot vxlan_bd_untagged.robot TC05,TC06
99 mgmt-cfg-acl-apihc-apivat-func.robot 080_access_control_lists.robot
100 mgmt-cfg-ietfacl-apihc-apivat-func.robot 081_ietf_acl_traffic.robot
101 mgmt-cfg-int-apihcnc-func.robot 070_netconf.robot
102 mgmt-cfg-int-subint-apihc-apivat-func.robot 060_sub_interface.robot
103 mgmt-cfg-intip4-intip6-apihc-apivat-func.robot 010_interface_management.robot
104 mgmt-cfg-inttap-apihc-apivat-func.robot 040_tap.robot
105 mgmt-cfg-inttap-apivat-func.robot tap_interface.robot TC06
106 mgmt-cfg-intvhost-apihc-apivat-func.robot 050_interface_vhost_user.robot
107 mgmt-cfg-l2bd-apihc-apivat-func.robot 020_bridge_domain.robot
108 mgmt-cfg-l2fib-apihc-apivat-func.robot 021_l2_fib.robot
109 mgmt-cfg-lisp-apihc-apivat-func.robot 100_lisp.robot
110 mgmt-cfg-lisp-apivat-func.robot api.robot TC01..TC05
111 mgmt-cfg-nsh-apihc-apivat-func.robot 110_nsh_sfc.robot
112 mgmt-cfg-pbb-apihc-apivat-func.robot 090_provider_backbone_bridge.robot
113 mgmt-cfg-snat44-apihc-apivat-func.robot 130_nat.robot
114 mgmt-cfg-spanrx-apihc-apivat-func.robot 120_port_mirroring.robot
115 mgmt-cfg-vxlan-apihc-apivat-func.robot 030_vxlan.robot
116 mgmt-cfg-vxlangpe-apihc-apivat-func.robot 031_vxlan_gpe.robot
117 mgmt-notif-apihcnc-func.robot 071_notification.robot
118 mgmt-statepersist-apihc-func.robot 900_persistence.robot
Retrieved from "https://wiki.fd.io/index.php?title=CSIT/csit-perf-tc-naming-change&oldid=4196"