From e71c66a5e9d46ac5c23c58e08b929a4eef2effbb Mon Sep 17 00:00:00 2001 From: Bin Yang Date: Fri, 27 Mar 2020 04:03:43 +0000 Subject: [PATCH] Update installation guide Update kubernetes version to 1.16.2 Add SRIOV CNI deployment instructions Issue-ID: INF-20 Signed-off-by: Bin Yang Change-Id: If286f3c7de98f6a2745def86da60ae573e70b04e --- docs/installation-guide.rst | 368 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 366 insertions(+), 2 deletions(-) diff --git a/docs/installation-guide.rst b/docs/installation-guide.rst index 5b6a817..4253326 100644 --- a/docs/installation-guide.rst +++ b/docs/installation-guide.rst @@ -187,7 +187,7 @@ This instruction will show you how to deploy kubernetes cluster and plugins in a :: - root@oran-aio:~# kubeadm init --kubernetes-version v1.15.2 --pod-network-cidr=10.244.0.0/16 + root@oran-aio:~# kubeadm init --kubernetes-version v1.16.2 --pod-network-cidr=10.244.0.0/16 root@oran-aio:~# mkdir -p $HOME/.kube root@oran-aio:~# cp -i /etc/kubernetes/admin.conf $HOME/.kube/config root@oran-aio:~# chown $(id -u):$(id -g) $HOME/.kube/config @@ -285,7 +285,371 @@ Verify that the node is labeled by nfd: feature.node.kubernetes.io/cpu-cpuid.AVX2=true (...snip...) -3.11 Deploy CMK (CPU-Manager-for-Kubernetes) +3.11 Deploy SRIOV CNI +''''''''''''''''''''' + +Provision VF drivers and devices + + +Enumerate PF Devices + +:: + + root@oran-aio:~/dpdk-18.08/usertools# lspci -D |grep 82599 + 0000:04:00.0 Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection (rev 01) + 0000:04:00.1 Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection (rev 01) + +Correlate the PF device to eth interfaces and bring them up + +:: + + root@oran-aio:~# ethtool -i eth4 |grep bus-info + bus-info: 0000:04:00.0 + root@oran-aio:~# ethtool -i eth5 |grep bus-info + bus-info: 0000:04:00.1 + root@oran-aio:~# ifconfig eth4 up + root@oran-aio:~# ifconfig eth5 up + +Load VF Driver modules + +:: + + root@oran-aio:~# modprobe ixgbevf + root@oran-aio:~# modprobe uio + root@oran-aio:~# modprobe igb-uio + root@oran-aio:~# modprobe vfio + root@oran-aio:~# modprobe vfio-pci + root@oran-aio:~# lsmod |grep ixgbevf + ixgbevf 61440 0 + root@oran-aio:~# lsmod |grep vfio + vfio_pci 40960 0 + vfio_virqfd 16384 1 vfio_pci + vfio_iommu_type1 24576 0 + vfio 24576 2 vfio_iommu_type1,vfio_pci + irqbypass 16384 2 vfio_pci,kvm + + +Bind VF drivers to VF devices + +:: + + root@oran-aio:~# cat /sys/bus/pci/devices/0000\:04\:00.0/sriov_totalvfs + root@oran-aio:~# cat /sys/bus/pci/devices/0000\:04\:00.1/sriov_totalvfs + root@oran-aio:~# cat /sys/bus/pci/devices/0000\:04\:00.0/sriov_numvfs + root@oran-aio:~# cat /sys/bus/pci/devices/0000\:04\:00.1/sriov_numvfs + root@oran-aio:~# echo 8 > /sys/bus/pci/devices/0000\:04\:00.0/sriov_numvfs + root@oran-aio:~# echo 8 > /sys/bus/pci/devices/0000\:04\:00.1/sriov_numvfs + + root@oran-aio:~# lspci -D |grep 82599 + 0000:04:00.0 Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection (rev 01) + 0000:04:00.1 Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection (rev 01) + 0000:04:10.0 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) + 0000:04:10.1 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) + 0000:04:10.2 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) + 0000:04:10.3 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) + 0000:04:10.4 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) + 0000:04:10.5 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) + 0000:04:10.6 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) + 0000:04:10.7 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) + 0000:04:11.0 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) + 0000:04:11.1 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) + 0000:04:11.2 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) + 0000:04:11.3 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) + 0000:04:11.4 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) + 0000:04:11.5 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) + 0000:04:11.6 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) + 0000:04:11.7 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01) + + root@oran-aio:~# dpdk-devbind -b vfio-pci 0000:04:11.0 0000:04:11.1 0000:04:11.2 0000:04:11.3 0000:04:11.4 0000:04:11.5 0000:04:11.6 0000:04:11.7 + + root@oran-aio:~# dpdk-devbind --status-dev net + + Network devices using DPDK-compatible driver + ============================================ + 0000:04:11.0 '82599 Ethernet Controller Virtual Function 10ed' drv=vfio-pci unused=ixgbevf,igb_uio + 0000:04:11.1 '82599 Ethernet Controller Virtual Function 10ed' drv=vfio-pci unused=ixgbevf,igb_uio + 0000:04:11.2 '82599 Ethernet Controller Virtual Function 10ed' drv=vfio-pci unused=ixgbevf,igb_uio + 0000:04:11.3 '82599 Ethernet Controller Virtual Function 10ed' drv=vfio-pci unused=ixgbevf,igb_uio + 0000:04:11.4 '82599 Ethernet Controller Virtual Function 10ed' drv=vfio-pci unused=ixgbevf,igb_uio + 0000:04:11.5 '82599 Ethernet Controller Virtual Function 10ed' drv=vfio-pci unused=ixgbevf,igb_uio + 0000:04:11.6 '82599 Ethernet Controller Virtual Function 10ed' drv=vfio-pci unused=ixgbevf,igb_uio + 0000:04:11.7 '82599 Ethernet Controller Virtual Function 10ed' drv=vfio-pci unused=ixgbevf,igb_uio + + Network devices using kernel driver + =================================== + 0000:04:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' if=eth4 drv=ixgbe unused=igb_uio,vfio-pci + 0000:04:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' if=eth5 drv=ixgbe unused=igb_uio,vfio-pci + 0000:04:10.0 '82599 Ethernet Controller Virtual Function 10ed' if=eth6 drv=ixgbevf unused=igb_uio,vfio-pci + 0000:04:10.1 '82599 Ethernet Controller Virtual Function 10ed' if=eth14 drv=ixgbevf unused=igb_uio,vfio-pci + 0000:04:10.2 '82599 Ethernet Controller Virtual Function 10ed' if=eth7 drv=ixgbevf unused=igb_uio,vfio-pci + 0000:04:10.3 '82599 Ethernet Controller Virtual Function 10ed' if=eth15 drv=ixgbevf unused=igb_uio,vfio-pci + 0000:04:10.4 '82599 Ethernet Controller Virtual Function 10ed' if=eth8 drv=ixgbevf unused=igb_uio,vfio-pci + 0000:04:10.5 '82599 Ethernet Controller Virtual Function 10ed' if=eth16 drv=ixgbevf unused=igb_uio,vfio-pci + 0000:04:10.6 '82599 Ethernet Controller Virtual Function 10ed' if= drv=ixgbevf unused=igb_uio,vfio-pci + 0000:04:10.7 '82599 Ethernet Controller Virtual Function 10ed' if=eth17 drv=ixgbevf unused=igb_uio,vfio-pci + + +Build SRIOV CNI + +:: + + root@oran-aio:~# HTTP_PROXY="http://:" + + root@oran-aio:~# wget https://dl.google.com/go/go1.14.1.linux-amd64.tar.gz + root@oran-aio:~# tar -zxvf go1.14.1.linux-amd64.tar.gz + root@oran-aio:~# PATH=$PATH:/root/go/bin/ + root@oran-aio:~# git clone https://github.com/intel/sriov-cni + root@oran-aio:~# cd sriov-cni + root@oran-aio:~/sriov-cni# make + root@oran-aio:~/sriov-cni# cp build/sriov /opt/cni/bin + + root@oran-aio:~# cd ~/ + root@oran-aio:~# git clone https://github.com/intel/sriov-network-device-plugin + root@oran-aio:~# cd sriov-network-device-plugin + root@oran-aio:~/sriov-network-device-plugin# git fetch origin pull/196/head:fpgadp + root@oran-aio:~/sriov-network-device-plugin# git checkout fpgadp + root@oran-aio:~/sriov-network-device-plugin# make image + root@oran-aio:~/sriov-network-device-plugin# docker images |grep sriov-device-plugin + nfvpe/sriov-device-plugin latest f4e6bbefad67 5 minutes ago 25.5MB + + +Deploy SRIOV CNI + +:: + + root@oran-aio:~/sriov-network-device-plugin# cat < deployments/sriovdp_configMap.yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: sriovdp-config + namespace: kube-system + data: + config.json: | + { + "resourceList": [{ + "resourceName": "intel_sriov_netdevice", + "selectors": { + "vendors": ["8086"], + "devices": ["154c", "10ed"], + "drivers": ["i40evf", "ixgbevf"] + } + }, + { + "resourceName": "intel_sriov_dpdk", + "selectors": { + "vendors": ["8086"], + "devices": ["154c", "10ed"], + "drivers": ["vfio-pci"] + } + }, + { + "resourceName": "mlnx_sriov_rdma", + "isRdma": true, + "selectors": { + "vendors": ["15b3"], + "devices": ["1018"], + "drivers": ["mlx5_ib"] + } + } + ] + } + EOF + + root@oran-aio:~/sriov-network-device-plugin# kubectl create -f deployments/sriovdp_configMap.yaml + root@oran-aio:~/sriov-network-device-plugin# kubectl create -f deployments/k8s-v1.16/sriovdp-daemonset.yaml + + root@oran-aio:~/sriov-network-device-plugin# kubectl get pods --all-namespaces |grep kube-sriov-device-plugin + kube-system kube-sriov-device-plugin-amd64-6lm8n 1/1 Running 0 12m + + root@oran-aio:~/sriov-network-device-plugin# kubectl -n kube-system logs kube-sriov-device-plugin-amd64-6lm8n + I0327 02:14:46.488409 14488 manager.go:115] Creating new ResourcePool: intel_sriov_netdevice + I0327 02:14:46.488427 14488 factory.go:144] device added: [pciAddr: 0000:04:10.0, vendor: 8086, device: 10ed, driver: ixgbevf] + I0327 02:14:46.488439 14488 factory.go:144] device added: [pciAddr: 0000:04:10.1, vendor: 8086, device: 10ed, driver: ixgbevf] + I0327 02:14:46.488446 14488 factory.go:144] device added: [pciAddr: 0000:04:10.2, vendor: 8086, device: 10ed, driver: ixgbevf] + I0327 02:14:46.488459 14488 factory.go:144] device added: [pciAddr: 0000:04:10.3, vendor: 8086, device: 10ed, driver: ixgbevf] + I0327 02:14:46.488467 14488 factory.go:144] device added: [pciAddr: 0000:04:10.4, vendor: 8086, device: 10ed, driver: ixgbevf] + I0327 02:14:46.488473 14488 factory.go:144] device added: [pciAddr: 0000:04:10.5, vendor: 8086, device: 10ed, driver: ixgbevf] + I0327 02:14:46.488479 14488 factory.go:144] device added: [pciAddr: 0000:04:10.6, vendor: 8086, device: 10ed, driver: ixgbevf] + I0327 02:14:46.488485 14488 factory.go:144] device added: [pciAddr: 0000:04:10.7, vendor: 8086, device: 10ed, driver: ixgbevf] + I0327 02:14:46.488502 14488 manager.go:128] New resource server is created for intel_sriov_netdevice ResourcePool + I0327 02:14:46.488511 14488 manager.go:114] + I0327 02:14:46.488516 14488 manager.go:115] Creating new ResourcePool: intel_sriov_dpdk + I0327 02:14:46.488529 14488 factory.go:144] device added: [pciAddr: 0000:04:11.0, vendor: 8086, device: 10ed, driver: vfio-pci] + I0327 02:14:46.488538 14488 factory.go:144] device added: [pciAddr: 0000:04:11.1, vendor: 8086, device: 10ed, driver: vfio-pci] + I0327 02:14:46.488545 14488 factory.go:144] device added: [pciAddr: 0000:04:11.2, vendor: 8086, device: 10ed, driver: vfio-pci] + I0327 02:14:46.488551 14488 factory.go:144] device added: [pciAddr: 0000:04:11.3, vendor: 8086, device: 10ed, driver: vfio-pci] + I0327 02:14:46.488562 14488 factory.go:144] device added: [pciAddr: 0000:04:11.4, vendor: 8086, device: 10ed, driver: vfio-pci] + I0327 02:14:46.488569 14488 factory.go:144] device added: [pciAddr: 0000:04:11.5, vendor: 8086, device: 10ed, driver: vfio-pci] + I0327 02:14:46.488575 14488 factory.go:144] device added: [pciAddr: 0000:04:11.6, vendor: 8086, device: 10ed, driver: vfio-pci] + I0327 02:14:46.488581 14488 factory.go:144] device added: [pciAddr: 0000:04:11.7, vendor: 8086, device: 10ed, driver: vfio-pci] + I0327 02:14:46.488591 14488 manager.go:128] New resource server is created for intel_sriov_dpdk ResourcePool + + +Test intel_sriov_netdeivce + +:: + + root@oran-aio:~/sriov-network-device-plugin# cat < deployments/sriov-crd.yaml + apiVersion: "k8s.cni.cncf.io/v1" + kind: NetworkAttachmentDefinition + metadata: + name: sriov-net1 + annotations: + k8s.v1.cni.cncf.io/resourceName: intel.com/intel_sriov_netdevice + spec: + config: '{ + "type": "sriov", + "cniVersion": "0.3.1", + "name": "sriov-network", + "vlan": 100, + "ipam": { + "type": "host-local", + "subnet": "10.56.217.0/24", + "routes": [{ + "dst": "0.0.0.0/0" + }], + "gateway": "10.56.217.1" + } + }' + EOF + + root@oran-aio:~/sriov-network-device-plugin# kubectl create -f deployments/sriov-crd.yaml + root@oran-aio:~/sriov-network-device-plugin# kubectl create -f deployments/pod-tc1.yaml + root@oran-aio:~/sriov-network-device-plugin# kubectl get pods |grep testpod1 + root@oran-aio:~/sriov-network-device-plugin# ip link |grep 'vlan 100' + vf 3 MAC a6:01:0a:34:39:e1, vlan 100, spoof checking on, link-state auto, trust off, query_rss off + + root@oran-aio:~/sriov-network-device-plugin# kubectl exec -it testpod1 -- ip addr show |grep a6:01:0a:34:39:e1 -C 2 + valid_lft forever preferred_lft forever + 21: net1: mtu 1500 qdisc mq state UP group default qlen 1000 + link/ether a6:01:0a:34:39:e1 brd ff:ff:ff:ff:ff:ff + inet 10.56.217.3/24 brd 10.56.217.255 scope global net1 + valid_lft forever preferred_lft forever + + +Test intel_sriov_dpdk + +:: + + root@oran-aio:~/sriov-network-device-plugin# cat < deployments/sriovdpdk-crd.yaml + apiVersion: "k8s.cni.cncf.io/v1" + kind: NetworkAttachmentDefinition + metadata: + name: sriov1-vfio + annotations: + k8s.v1.cni.cncf.io/resourceName: intel.com/intel_sriov_dpdk + spec: + config: '{ + "type": "sriov", + "cniVersion": "0.3.1", + "vlan": 101, + "name": "sriov1-vfio" + }' + EOF + + root@oran-aio:~/sriov-network-device-plugin# cat < deployments/dpdk-1g.yaml + apiVersion: v1 + kind: Pod + metadata: + name: dpdk-1g + annotations: + k8s.v1.cni.cncf.io/networks: '[ + {"name": "sriov1-vfio"}, + {"name": "sriov1-vfio"} + ]' + spec: + restartPolicy: Never + containers: + - name: dpdk-1g + image: centos/tools + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /mnt/huge-2048 + name: hugepage + - name: lib-modules + mountPath: /lib/modules + - name: src + mountPath: /usr/src + command: ["/bin/bash", "-ec", "sleep infinity"] + securityContext: + privileged: true + capabilities: + add: + - ALL + resources: + requests: + memory: 4Gi + hugepages-1Gi: 4Gi + intel.com/intel_sriov_dpdk: '2' + limits: + memory: 4Gi + hugepages-1Gi: 4Gi + intel.com/intel_sriov_dpdk: '2' + imagePullSecrets: + - name: admin-registry-secret + volumes: + - name: hugepage + emptyDir: + medium: HugePages + - name: lib-modules + hostPath: + path: /lib/modules + - name: src + hostPath: + path: /usr/src + imagePullSecrets: + - name: admin-registry-secret + EOF + + root@oran-aio:~/sriov-network-device-plugin# kubectl create -f deployments/sriovdpdk-crd.yaml + root@oran-aio:~/sriov-network-device-plugin# kubectl create -f deployments/dpdk-1g.yaml + + root@oran-aio:~/sriov-network-device-plugin# root@oran-aio:~/sriov-network-device-plugin# kubectl get pods | grep dpdk + dpdk-1g 1/1 Running 0 13s + + root@oran-aio:~/sriov-network-device-plugin# ip link |grep 101 + vf 7 MAC 00:00:00:00:00:00, vlan 101, spoof checking on, link-state auto, trust off, query_rss off + vf 6 MAC 00:00:00:00:00:00, vlan 101, spoof checking on, link-state auto, trust off, query_rss off + + +Now test with dpdk + +:: + + ### build following package and copy to target server: bitbake bison;bitbake kernel-devsrc + root@oran-aio:~/sriov-network-device-plugin# rpm -ivh ~/bison-3.0.4-r0.corei7_64.rpm + root@oran-aio:~/sriov-network-device-plugin# rpm -ivh ~/kernel-devsrc-1.0-r0.intel_x86_64.rpm + + root@oran-aio:~/sriov-network-device-plugin# kubectl exec -it $(kubectl get pods -o wide | grep dpdk | awk '{ print $1 }') -- /bin/bash + [root@dpdk-1g /]# export |grep INTEL + declare -x PCIDEVICE_INTEL_COM_INTEL_SRIOV_DPDK="0000:04:11.6,0000:04:11.5" + + [root@dpdk-1g /]# yum -y install wget ncurses-devel unzip libpcap-devel ncurses-devel libedit-devel pciutils lua-devel + + [root@dpdk-1g /]# cd /opt + [root@dpdk-1g /]# wget https://fast.dpdk.org/rel/dpdk-18.08.tar.xz + [root@dpdk-1g /]# tar xf dpdk-18.08.tar.xz + [root@dpdk-1g /]# cd dpdk-18.08/ + [root@dpdk-1g /]# sed -i 's/CONFIG_RTE_EAL_IGB_UIO=y/CONFIG_RTE_EAL_IGB_UIO=n/g' config/common_linuxapp + [root@dpdk-1g /]# sed -i 's/CONFIG_RTE_LIBRTE_KNI=y/CONFIG_RTE_LIBRTE_KNI=n/g' config/common_linuxapp + [root@dpdk-1g /]# sed -i 's/CONFIG_RTE_KNI_KMOD=y/CONFIG_RTE_KNI_KMOD=n/g' config/common_linuxapp + [root@dpdk-1g /]# export RTE_SDK=/opt/dpdk-18.08 + [root@dpdk-1g /]# export RTE_TARGET=x86_64-native-linuxapp-gcc + [root@dpdk-1g /]# export RTE_BIND=$RTE_SDK/usertools/dpdk-devbind.py + [root@dpdk-1g /]# make install T=$RTE_TARGET + [root@dpdk-1g /]# cd examples/helloworld + [root@dpdk-1g /]# make + [root@dpdk-1g /]# NR_hugepages=2 + [root@dpdk-1g /]# ./build/helloworld -l 1-4 -n 4 -m $NR_hugepages + ... + hello from core 2 + hello from core 3 + hello from core 4 + hello from core 1 + + + +3.12 Deploy CMK (CPU-Manager-for-Kubernetes) '''''''''''''''''''''''''''''''''''''''''''' Build the CMK docker image -- 2.16.6