Improve perf image and performance playbooks

* Generate a new image with latest rhel and drivers
* Use dpdk packages instead of compiling dpdk (update playbooks and images)
* Do not use old commands like dpdk-devbind.py, use driverctl instead
* Do not use io mode in testpmd, use mac mode instead. According to OSPRH-13653,
  it provides non realistic results.
* Vms configuration is permantent, so they can be rebooted and
  configuration will persist and performance can be run with any other
  configuration needed

Author: mnietoji

roles/images/performance/defaults/main.yml

@@ -1,24 +1,16 @@
 # Image Variables
-guest_image: 'http://download.eng.tlv.redhat.com/rhel-9/rel-eng/RHEL-9/latest-RHEL-9.2.0/compose/BaseOS/x86_64/images/rhel-guest-image-9.2-20230414.17.x86_64.qcow2'
+guest_image: 'http://download.hosts.prod.upshift.rdu2.redhat.com/rhel-9/rel-eng/RHEL-9/latest-RHEL-9.5.0/compose/BaseOS/x86_64/images/rhel-guest-image-9.5-20241009.2.x86_64.qcow2'
 guest_image_output: '/tmp/'
 guest_debug: False
 
 # Repo Variables
 repo_urls:
-  - http://download.eng.tlv.redhat.com/rhel-9/rel-eng/RHEL-9/latest-RHEL-9.2.0/repofile.repo
+  - http://download.hosts.prod.upshift.rdu2.redhat.com/rhel-9/rel-eng/RHEL-9/latest-RHEL-9.5.0/repofile.repo
 repo_dir: '/tmp/guest_repos'
 guest_repos: []
 
-# DPDK Variables
-dpdk_dir: '/root/dpdk'
-dpdk_customization_log: "{{ dpdk_dir }}/customization.log"
-dpdk_git: 'git://dpdk.org/dpdk'
-dpdk_branch: 'tags/v22.11'
-dpdk_compile_packages:
-  - pyelftools
-
 # Trex Variables
-trex_version: 'v3.02'
+trex_version: 'v3.06'
 trex_url: 'https://trex-tgn.cisco.com/trex/release'
 trex_dir: '/opt/trex/'
 
@@ -41,7 +33,3 @@ fio_git: 'git://git.kernel.dk/fio.git'
 fio_dir: '/root/fio'
 fio_branch: 'fio-3.9'
 fio_customization_log: "{{ fio_dir }}/customization.log"
-
-# mellanox drivers
-# mlx_iso: http://file.mad.redhat.com/~mnietoji/MLNX_OFED_LINUX-5.9-0.5.6.0-rhel9.1-x86_64.iso
-mlx_iso: ''

roles/images/performance/tasks/generic_customizations.yml

@@ -2,24 +2,14 @@
   command: virt-customize -a {{ fetched_image }} --run-command "yum install {{ item }} -y"
   loop:
     - git
-    - gcc
-    - make
-    - cmake
-    - gcc-c++
-    - pciutils
-    - glibc-devel
-    - glibc-headers
-    - numactl-devel
     - vim
     - tmux
     - python3
     - cloud-init
-    - meson
     - pip
     - iperf3
     - mstflint
     - tuned-profiles-cpu-partitioning
-    - perl
     - net-tools
 
 - name: create custom users

roles/images/performance/tasks/prepare_dpdk.yml

@@ -1,37 +1,12 @@
-- name: install dpdk compile dependency
-  command: virt-customize -a {{ fetched_image }} --run-command "pip3 install {{ item }}"
-  loop: "{{ dpdk_compile_packages }}"
-
-- name: configure mellanox drivers
-  block:
-  - name: create mellanox installation script
-    lineinfile:
-      dest: /tmp/mlx_script.sh
-      line: '{{ item }}'
-      create: true
-      mode: 0744
-    with_items:
-      - 'curl -o /tmp/{{ mlx_iso | basename }} {{ mlx_iso }}'
-      - 'mount -o ro,loop /tmp/{{ mlx_iso | basename }} /mnt'
-      - 'cat <<EOF >> /etc/yum.repos.d/mlnx_ofed.repo'
-      - '[mlnx_ofed]'
-      - 'name=MLNX_OFED Repository'
-      - 'baseurl=file:///mnt/RPMS'
-      - 'enabled=1'
-      - 'gpgcheck=0'
-      - 'EOF'
-      - 'yum install mlnx-ofed-all -y'
-      - 'rm /etc/yum.repos.d/mlnx_ofed.repo'
-
-  - name: install mellanox drivers
-    command: virt-customize -v -x  -a {{ fetched_image }} --run /tmp/mlx_script.sh
-  when: mlx_iso | length > 0
+- name: Install dpdk
+  command: virt-customize -a {{ fetched_image }} --run-command "yum install {{ item }} -y"
+  loop:
+    - dpdk
+    - driverctl
+    - pciutils
 
-- name: compile dpdk and install trafficgen
+- name: install trafficgen
   command: virt-customize -a {{ fetched_image }} --run-command "{{ item }}"
   loop:
-    - "if [ ! -d {{ dpdk_dir }} ];then git clone {{ dpdk_git }} {{ dpdk_dir }};cd {{ dpdk_dir }};git checkout {{ dpdk_branch }};fi"
     - "if [ ! -d {{ trafficgen_dir }} ];then git clone {{ trafficgen_git }} {{ trafficgen_dir }};cd {{ trafficgen_dir }};git checkout {{ trafficgen_branch }};fi"
     - "sed -i 's/enforcing/{{ selinux_config }}/g' /etc/selinux/config /etc/selinux/config"
-    - "cd {{ dpdk_dir }} ; meson setup build; ninja -C build"
-    - "echo 'DPDK Version: {{ dpdk_branch }}' | tee {{ dpdk_customization_log }}"

roles/packet_gen/trex/README.md

@@ -116,9 +116,6 @@ binary_search: True
 States if config variables should be parsed during run time, if `False`, a user must pass the required trex configuration variables.
 
 `True` by default.
-```
-gather_trex_conf_live: True
-```
 
 Destination out trex config file which will be created on trex instance.
 
@@ -203,25 +200,6 @@ trafficgen_branch: 'master'
 ```
 
 ### DPDK Variables
-
-DPDK root directory on instances(cloned from: 'git://dpdk.org/dpdk').
-
-`/root/dpdk` by default.
-```
-dpdk_root_dir: '/root/dpdk'
-```
-
-Compiled DPDK binaries directory on instances(compiled as part of NFV perf guest image).
-
-Info about what we compile can be found in `prepare_performance_images` [role documentation](/roles/images/performance/README.md#description).
-
-Compilation steps can be found inside `Prepare DPDK binaries inside guest image` [task](/roles/images/performance/tasks/prepare_dpdk.yml).
-
-`{{ dpdk_root_dir }}/build/app` by default.
-```
-dpdk_compiled_dir: '/root/dpdk/build/app'
-```
-
 Binding drivers used by dpdk (vfio-pci, mlx5_core)
 'vfio-pci' by default
 

roles/packet_gen/trex/binary_search/defaults/main.yml

@@ -2,14 +2,17 @@
 binary_perf_log: '/tmp/performance.log'
 
 # Trafficgen repo directory on host
-trafficgen_dir: '/opt/trafficgen'
+trafficgen_dir: '/opt/bench-trafficgen/trafficgen'
 
 # binary-search binary
 binary_search_bin: "{{ trafficgen_dir }}/binary-search.py"
 
 # Frame size in bytes that Trex will generate
 trex_frame_size: 64
 
+# Destination ports, one per device
+dst_ports: 53000,54000
+
 # Max packet lost % acceptable
 trex_max_lost_pct: 0.00
 
@@ -68,4 +71,4 @@ binary_search_status_delay: 10
 
 # What to do when negative packet loss is encountered
 # fail,quit,retry-to-fail,retry-to-quit
-negative_packet_loss: retry-to-fail
+negative_packet_loss: retry-to-fail

roles/packet_gen/trex/binary_search/tasks/main.yml

@@ -34,6 +34,7 @@
       --max-loss-pct {{ trex_max_lost_pct }}
       --send-teaching-warmup
       --dst-macs {{ dut_macs }}
+      --dst-ports {{ dst_ports }}
       --num-flows {{ trex_flows }}
       {% if trex_set_vlan %}
         --vlan-ids {{ trex_vlans }}

roles/packet_gen/trex/bind_dpdk_nics/defaults/main.yml

@@ -1,9 +1,3 @@
-# DPDK directory on host
-dpdk_root_dir: '/root/dpdk'
-
-# Compiled DPDK binaries directory
-dpdk_compiled_dir: '{{ dpdk_root_dir }}/build/app'
-
 hugepages_count: 4
 
 # dpdk drivers to use: vfio-pci, mlx5_core

roles/packet_gen/trex/bind_dpdk_nics/tasks/main.yml

@@ -1,10 +1,3 @@
-- name: Check if DPDK Compiled Binaries Directory Exists
-  stat:
-    path: "{{ dpdk_compiled_dir }}"
-  register: dpdk_dir
-  failed_when: not dpdk_dir['stat']['exists']
-  become: True
-
 - name: Retrieve default NIC
   set_fact:
     nic_def: "{{hostvars[inventory_hostname]['ansible_default_ipv4']['alias']}}"
@@ -22,8 +15,11 @@
   loop: "{{nic_down}}"
   become: True
 
-- name: Retrieve Non Active NICs' PCI Slots
-  shell: "python {{ dpdk_root_dir }}/usertools/dpdk-devbind.py --status | grep -A 5 'Network devices using kernel driver' | grep -v Active | grep 0000 | cut -d ' ' -f 1 | paste -sd ' '"
+- name: Retrieve pci slots for nics
+  shell: |
+    for nic in {{ nic_down | join(' ') }}; do
+      ethtool -i $nic | awk '{if ($1 == "bus-info:") print $2}'
+    done
   register: instance_nics_pci_slots
   become: True
 
@@ -54,21 +50,21 @@
         name: "{{ item['name'] }}"
         state: present
         params: "{{ item['params'] | default(omit) }}"
+        persistent: "present"
       loop:
-        - {'name': 'vfio'}
+        - {'name': 'vfio', 'params': 'enable_unsafe_noiommu_mode=1'}
         - {'name': 'vfio-pci'}
 
-    - name: Ensure enable_unsafe_noiommu_mode
-      shell: "echo 1 | sudo tee /sys/module/vfio/parameters/enable_unsafe_noiommu_mode"
-
     - name: Load Mellanox Kernel Modules when configured
       modprobe:
         name: mlx5_core
         state: present
+        persistent: "present"
       when: dpdk_binding_driver == "mlx5_core"
 
     - name: Bind DPDK NICs
-      command: "{{ dpdk_root_dir }}/usertools/dpdk-devbind.py -b {{dpdk_binding_driver}} {{ instance_nics_pci_slots['stdout'] }}"
+      command: "driverctl set-override {{ item }} {{dpdk_binding_driver}}"
+      loop: "{{ instance_nics_pci_slots.stdout_lines }}"
 
   become: True
 

roles/packet_gen/trex/launch_testpmd/defaults/main.yml

@@ -1,5 +1,5 @@
 # testpmd binary
-testpmd_bin: "{{ dpdk_compiled_dir }}/testpmd"
+testpmd_bin: "/usr/bin/dpdk-testpmd"
 
 # Map list of cores to physical cpu set
 testpmd_lcores: '3,4,7'
@@ -26,10 +26,13 @@ testpmd_rxq: 1
 testpmd_txq: 1
 
 # TestPMD forwarding mode
-forward_mode: io
+forward_mode: mac
 
 # TestPMD verbose
 testpmd_verbose: 0
 
 # TestPMD log file
-testpmd_log: "/tmp/testpmd.log"
+testpmd_log: "/tmp/testpmd.log"
+
+# Enable IP forwarding in case L3 needed
+packet_gen_ip_forwarding: false

roles/packet_gen/trex/trex_instance_config/defaults/main.yaml

@@ -1,21 +1,6 @@
-# Parse trex configuration during run time from trex instance
-gather_trex_conf_live: True
-
 # Destination of trex config file
 trex_conf_file: '/etc/trex_cfg.yaml'
 
-# Clone trafficgen repo https://github.com/atheurer/trafficgen
-clone_traffic_gen_repo: True
-
-# Trafficgen repo URI
-trafficgen_repo: 'https://github.com/atheurer/trafficgen'
-
-# Trafficgen branch
-trafficgen_branch: 'master'
-
-# Derecitory where trafficgen repo will be cloned to
-trafficgen_dir: '/opt/trafficgen/'
-
 # Platform sepcific Trex configuration
 trex_platform:
   master_thread_id: 2
@@ -24,8 +9,5 @@ trex_platform:
     - socket: 0
       threads: [4, 5, 6, 7, 8, 9, 10, 11]
 
-# DPDK root dir
-dpdk_root_dir: '/root/dpdk'
-
 # Workaround - can't define this variable inside playbook
 trex_full_nic_info: []