-- For detail usage, please refer to `Doc for dashboard`_
-
-.. _`Doc for dashboard`: https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/
-
-3.9 Deploy Multus-CNI
-'''''''''''''''''''''
-
-::
-
- root@oran-aio:~# kubectl apply -f /etc/kubernetes/plugins/multus-cni/multus-daemonset.yml
-
-Verify that the multus-cni is up and running
-
-::
-
- root@oran-aio:~# kubectl get pods --all-namespaces | grep -i multus
- kube-system kube-multus-ds-amd64-hjpk4 1/1 Running 0 7m34s
-
-- For further validating, please refer to the `Multus-CNI quick start`_
-
-.. _`Multus-CNI quick start`: https://github.com/intel/multus-cni/blob/master/doc/quickstart.md
-
-3.10 Deploy NFD (node-feature-discovery)
-''''''''''''''''''''''''''''''''''''''''
-
-::
-
- root@oran-aio:~# kubectl apply -f /etc/kubernetes/plugins/node-feature-discovery/nfd-master.yaml
- root@oran-aio:~# kubectl apply -f /etc/kubernetes/plugins/node-feature-discovery/nfd-worker-daemonset.yaml
-
-Verify that nfd-master and nfd-worker are up and running
-
-::
-
- root@oran-aio:~# kubectl get pods --all-namespaces |grep nfd
- default nfd-master-7v75k 1/1 Running 0 91s
- default nfd-worker-xn797 1/1 Running 0 24s
-
-Verify that the node is labeled by nfd:
-
-::
-
- root@oran-aio:~# kubectl describe nodes|grep feature.node.kubernetes
- feature.node.kubernetes.io/cpu-cpuid.AESNI=true
- feature.node.kubernetes.io/cpu-cpuid.AVX=true
- feature.node.kubernetes.io/cpu-cpuid.AVX2=true
- (...snip...)
-
-3.11 Deploy SRIOV CNI
-'''''''''''''''''''''
-
-Provision VF drivers and devices
-
-
-Enumerate PF Devices
-
-::
-
- root@oran-aio:~/dpdk-18.08/usertools# lspci -D |grep 82599
- 0000:04:00.0 Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection (rev 01)
- 0000:04:00.1 Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection (rev 01)
-
-Correlate the PF device to eth interfaces and bring them up
-
-::
-
- root@oran-aio:~# ethtool -i eth4 |grep bus-info
- bus-info: 0000:04:00.0
- root@oran-aio:~# ethtool -i eth5 |grep bus-info
- bus-info: 0000:04:00.1
- root@oran-aio:~# ifconfig eth4 up
- root@oran-aio:~# ifconfig eth5 up
-
-Load VF Driver modules
-
-::
-
- root@oran-aio:~# modprobe ixgbevf
- root@oran-aio:~# modprobe uio
- root@oran-aio:~# modprobe igb-uio
- root@oran-aio:~# modprobe vfio
- root@oran-aio:~# modprobe vfio-pci
- root@oran-aio:~# lsmod |grep ixgbevf
- ixgbevf 61440 0
- root@oran-aio:~# lsmod |grep vfio
- vfio_pci 40960 0
- vfio_virqfd 16384 1 vfio_pci
- vfio_iommu_type1 24576 0
- vfio 24576 2 vfio_iommu_type1,vfio_pci
- irqbypass 16384 2 vfio_pci,kvm
-
-
-Bind VF drivers to VF devices
-
-::
-
- root@oran-aio:~# cat /sys/bus/pci/devices/0000\:04\:00.0/sriov_totalvfs
- root@oran-aio:~# cat /sys/bus/pci/devices/0000\:04\:00.1/sriov_totalvfs
- root@oran-aio:~# cat /sys/bus/pci/devices/0000\:04\:00.0/sriov_numvfs
- root@oran-aio:~# cat /sys/bus/pci/devices/0000\:04\:00.1/sriov_numvfs
- root@oran-aio:~# echo 8 > /sys/bus/pci/devices/0000\:04\:00.0/sriov_numvfs
- root@oran-aio:~# echo 8 > /sys/bus/pci/devices/0000\:04\:00.1/sriov_numvfs
-
- root@oran-aio:~# lspci -D |grep 82599
- 0000:04:00.0 Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection (rev 01)
- 0000:04:00.1 Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection (rev 01)
- 0000:04:10.0 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
- 0000:04:10.1 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
- 0000:04:10.2 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
- 0000:04:10.3 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
- 0000:04:10.4 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
- 0000:04:10.5 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
- 0000:04:10.6 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
- 0000:04:10.7 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
- 0000:04:11.0 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
- 0000:04:11.1 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
- 0000:04:11.2 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
- 0000:04:11.3 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
- 0000:04:11.4 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
- 0000:04:11.5 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
- 0000:04:11.6 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
- 0000:04:11.7 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
-
- root@oran-aio:~# dpdk-devbind -b vfio-pci 0000:04:11.0 0000:04:11.1 0000:04:11.2 0000:04:11.3 0000:04:11.4 0000:04:11.5 0000:04:11.6 0000:04:11.7
-
- root@oran-aio:~# dpdk-devbind --status-dev net
-
- Network devices using DPDK-compatible driver
- ============================================
- 0000:04:11.0 '82599 Ethernet Controller Virtual Function 10ed' drv=vfio-pci unused=ixgbevf,igb_uio
- 0000:04:11.1 '82599 Ethernet Controller Virtual Function 10ed' drv=vfio-pci unused=ixgbevf,igb_uio
- 0000:04:11.2 '82599 Ethernet Controller Virtual Function 10ed' drv=vfio-pci unused=ixgbevf,igb_uio
- 0000:04:11.3 '82599 Ethernet Controller Virtual Function 10ed' drv=vfio-pci unused=ixgbevf,igb_uio
- 0000:04:11.4 '82599 Ethernet Controller Virtual Function 10ed' drv=vfio-pci unused=ixgbevf,igb_uio
- 0000:04:11.5 '82599 Ethernet Controller Virtual Function 10ed' drv=vfio-pci unused=ixgbevf,igb_uio
- 0000:04:11.6 '82599 Ethernet Controller Virtual Function 10ed' drv=vfio-pci unused=ixgbevf,igb_uio
- 0000:04:11.7 '82599 Ethernet Controller Virtual Function 10ed' drv=vfio-pci unused=ixgbevf,igb_uio
-
- Network devices using kernel driver
- ===================================
- 0000:04:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' if=eth4 drv=ixgbe unused=igb_uio,vfio-pci
- 0000:04:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' if=eth5 drv=ixgbe unused=igb_uio,vfio-pci
- 0000:04:10.0 '82599 Ethernet Controller Virtual Function 10ed' if=eth6 drv=ixgbevf unused=igb_uio,vfio-pci
- 0000:04:10.1 '82599 Ethernet Controller Virtual Function 10ed' if=eth14 drv=ixgbevf unused=igb_uio,vfio-pci
- 0000:04:10.2 '82599 Ethernet Controller Virtual Function 10ed' if=eth7 drv=ixgbevf unused=igb_uio,vfio-pci
- 0000:04:10.3 '82599 Ethernet Controller Virtual Function 10ed' if=eth15 drv=ixgbevf unused=igb_uio,vfio-pci
- 0000:04:10.4 '82599 Ethernet Controller Virtual Function 10ed' if=eth8 drv=ixgbevf unused=igb_uio,vfio-pci
- 0000:04:10.5 '82599 Ethernet Controller Virtual Function 10ed' if=eth16 drv=ixgbevf unused=igb_uio,vfio-pci
- 0000:04:10.6 '82599 Ethernet Controller Virtual Function 10ed' if= drv=ixgbevf unused=igb_uio,vfio-pci
- 0000:04:10.7 '82599 Ethernet Controller Virtual Function 10ed' if=eth17 drv=ixgbevf unused=igb_uio,vfio-pci
-
-
-Build SRIOV CNI
-
-::
-
- root@oran-aio:~# HTTP_PROXY="http://<your_proxy_server_ip>:<port>"
-
- root@oran-aio:~# wget https://dl.google.com/go/go1.14.1.linux-amd64.tar.gz
- root@oran-aio:~# tar -zxvf go1.14.1.linux-amd64.tar.gz
- root@oran-aio:~# PATH=$PATH:/root/go/bin/
- root@oran-aio:~# git clone https://github.com/intel/sriov-cni
- root@oran-aio:~# cd sriov-cni
- root@oran-aio:~/sriov-cni# make
- root@oran-aio:~/sriov-cni# cp build/sriov /opt/cni/bin
-
- root@oran-aio:~# cd ~/
- root@oran-aio:~# git clone https://github.com/intel/sriov-network-device-plugin
- root@oran-aio:~# cd sriov-network-device-plugin
- root@oran-aio:~/sriov-network-device-plugin# git fetch origin pull/196/head:fpgadp
- root@oran-aio:~/sriov-network-device-plugin# git checkout fpgadp
- root@oran-aio:~/sriov-network-device-plugin# make image
- root@oran-aio:~/sriov-network-device-plugin# docker images |grep sriov-device-plugin
- nfvpe/sriov-device-plugin latest f4e6bbefad67 5 minutes ago 25.5MB
-
-
-Deploy SRIOV CNI
-
-::
-
- root@oran-aio:~/sriov-network-device-plugin# cat <<EOF> deployments/sriovdp_configMap.yaml
- apiVersion: v1
- kind: ConfigMap
- metadata:
- name: sriovdp-config
- namespace: kube-system
- data:
- config.json: |
- {
- "resourceList": [{
- "resourceName": "intel_sriov_netdevice",
- "selectors": {
- "vendors": ["8086"],
- "devices": ["154c", "10ed"],
- "drivers": ["i40evf", "ixgbevf"]
- }
- },
- {
- "resourceName": "intel_sriov_dpdk",
- "selectors": {
- "vendors": ["8086"],
- "devices": ["154c", "10ed"],
- "drivers": ["vfio-pci"]
- }
- },
- {
- "resourceName": "mlnx_sriov_rdma",
- "isRdma": true,
- "selectors": {
- "vendors": ["15b3"],
- "devices": ["1018"],
- "drivers": ["mlx5_ib"]
- }
- }
- ]
- }
- EOF
-
- root@oran-aio:~/sriov-network-device-plugin# kubectl create -f deployments/sriovdp_configMap.yaml
- root@oran-aio:~/sriov-network-device-plugin# kubectl create -f deployments/k8s-v1.16/sriovdp-daemonset.yaml
-
- root@oran-aio:~/sriov-network-device-plugin# kubectl get pods --all-namespaces |grep kube-sriov-device-plugin
- kube-system kube-sriov-device-plugin-amd64-6lm8n 1/1 Running 0 12m
-
- root@oran-aio:~/sriov-network-device-plugin# kubectl -n kube-system logs kube-sriov-device-plugin-amd64-6lm8n
- I0327 02:14:46.488409 14488 manager.go:115] Creating new ResourcePool: intel_sriov_netdevice
- I0327 02:14:46.488427 14488 factory.go:144] device added: [pciAddr: 0000:04:10.0, vendor: 8086, device: 10ed, driver: ixgbevf]
- I0327 02:14:46.488439 14488 factory.go:144] device added: [pciAddr: 0000:04:10.1, vendor: 8086, device: 10ed, driver: ixgbevf]
- I0327 02:14:46.488446 14488 factory.go:144] device added: [pciAddr: 0000:04:10.2, vendor: 8086, device: 10ed, driver: ixgbevf]
- I0327 02:14:46.488459 14488 factory.go:144] device added: [pciAddr: 0000:04:10.3, vendor: 8086, device: 10ed, driver: ixgbevf]
- I0327 02:14:46.488467 14488 factory.go:144] device added: [pciAddr: 0000:04:10.4, vendor: 8086, device: 10ed, driver: ixgbevf]
- I0327 02:14:46.488473 14488 factory.go:144] device added: [pciAddr: 0000:04:10.5, vendor: 8086, device: 10ed, driver: ixgbevf]
- I0327 02:14:46.488479 14488 factory.go:144] device added: [pciAddr: 0000:04:10.6, vendor: 8086, device: 10ed, driver: ixgbevf]
- I0327 02:14:46.488485 14488 factory.go:144] device added: [pciAddr: 0000:04:10.7, vendor: 8086, device: 10ed, driver: ixgbevf]
- I0327 02:14:46.488502 14488 manager.go:128] New resource server is created for intel_sriov_netdevice ResourcePool
- I0327 02:14:46.488511 14488 manager.go:114]
- I0327 02:14:46.488516 14488 manager.go:115] Creating new ResourcePool: intel_sriov_dpdk
- I0327 02:14:46.488529 14488 factory.go:144] device added: [pciAddr: 0000:04:11.0, vendor: 8086, device: 10ed, driver: vfio-pci]
- I0327 02:14:46.488538 14488 factory.go:144] device added: [pciAddr: 0000:04:11.1, vendor: 8086, device: 10ed, driver: vfio-pci]
- I0327 02:14:46.488545 14488 factory.go:144] device added: [pciAddr: 0000:04:11.2, vendor: 8086, device: 10ed, driver: vfio-pci]
- I0327 02:14:46.488551 14488 factory.go:144] device added: [pciAddr: 0000:04:11.3, vendor: 8086, device: 10ed, driver: vfio-pci]
- I0327 02:14:46.488562 14488 factory.go:144] device added: [pciAddr: 0000:04:11.4, vendor: 8086, device: 10ed, driver: vfio-pci]
- I0327 02:14:46.488569 14488 factory.go:144] device added: [pciAddr: 0000:04:11.5, vendor: 8086, device: 10ed, driver: vfio-pci]
- I0327 02:14:46.488575 14488 factory.go:144] device added: [pciAddr: 0000:04:11.6, vendor: 8086, device: 10ed, driver: vfio-pci]
- I0327 02:14:46.488581 14488 factory.go:144] device added: [pciAddr: 0000:04:11.7, vendor: 8086, device: 10ed, driver: vfio-pci]
- I0327 02:14:46.488591 14488 manager.go:128] New resource server is created for intel_sriov_dpdk ResourcePool
-
-
-Test intel_sriov_netdeivce
-
-::
-
- root@oran-aio:~/sriov-network-device-plugin# cat <<EOF> deployments/sriov-crd.yaml
- apiVersion: "k8s.cni.cncf.io/v1"
- kind: NetworkAttachmentDefinition
- metadata:
- name: sriov-net1
- annotations:
- k8s.v1.cni.cncf.io/resourceName: intel.com/intel_sriov_netdevice
- spec:
- config: '{
- "type": "sriov",
- "cniVersion": "0.3.1",
- "name": "sriov-network",
- "vlan": 100,
- "ipam": {
- "type": "host-local",
- "subnet": "10.56.217.0/24",
- "routes": [{
- "dst": "0.0.0.0/0"
- }],
- "gateway": "10.56.217.1"
- }
- }'
- EOF
-
- root@oran-aio:~/sriov-network-device-plugin# kubectl create -f deployments/sriov-crd.yaml
- root@oran-aio:~/sriov-network-device-plugin# kubectl create -f deployments/pod-tc1.yaml
- root@oran-aio:~/sriov-network-device-plugin# kubectl get pods |grep testpod1
- root@oran-aio:~/sriov-network-device-plugin# ip link |grep 'vlan 100'
- vf 3 MAC a6:01:0a:34:39:e1, vlan 100, spoof checking on, link-state auto, trust off, query_rss off
-
- root@oran-aio:~/sriov-network-device-plugin# kubectl exec -it testpod1 -- ip addr show |grep a6:01:0a:34:39:e1 -C 2
- valid_lft forever preferred_lft forever
- 21: net1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
- link/ether a6:01:0a:34:39:e1 brd ff:ff:ff:ff:ff:ff
- inet 10.56.217.3/24 brd 10.56.217.255 scope global net1
- valid_lft forever preferred_lft forever