From 121f1b19ff1e1ea49144791b7bbb62009f1cfa0e Mon Sep 17 00:00:00 2001 From: Bin Yang Date: Fri, 27 Mar 2020 02:30:11 +0000 Subject: [PATCH] Upgrade kubernetes version to 1.16.2 Upgrade kubenetes plugins version as well Issue-ID: INF-20 Signed-off-by: Bin Yang Change-Id: Ia942eeca2ef0139a81fbb278be180f55e6e32e45 --- meta-oran/conf/distro/oran-inf.conf | 2 +- .../kubernetes-plugins/flannel_0.12.0.bb | 35 +++++++ .../kubernetes-dashboard_1.10.1.bb | 35 +++++++ .../kubernetes-plugins/multus-cni_3.3.bb | 3 +- .../node-feature-discovery_0.5.0.bb | 49 +++++++++ .../kubernetes/kubernetes/kubeadm.conf | 17 +++ .../kubernetes/kubernetes/kubelet-cgroup-setup.sh | 115 +++++++++++++++++++++ .../kubernetes/kubernetes-accounting.conf | 4 + .../kubernetes/kubernetes_1.16.2.bb | 42 ++++++++ .../feature/hosts-ia/files/other_drivers.cfg | 2 + 10 files changed, 302 insertions(+), 2 deletions(-) create mode 100644 meta-oran/recipes-containers/kubernetes-plugins/flannel_0.12.0.bb create mode 100644 meta-oran/recipes-containers/kubernetes-plugins/kubernetes-dashboard_1.10.1.bb create mode 100644 meta-oran/recipes-containers/kubernetes-plugins/node-feature-discovery_0.5.0.bb create mode 100644 meta-oran/recipes-containers/kubernetes/kubernetes/kubeadm.conf create mode 100644 meta-oran/recipes-containers/kubernetes/kubernetes/kubelet-cgroup-setup.sh create mode 100644 meta-oran/recipes-containers/kubernetes/kubernetes/kubernetes-accounting.conf create mode 100644 meta-oran/recipes-containers/kubernetes/kubernetes_1.16.2.bb diff --git a/meta-oran/conf/distro/oran-inf.conf b/meta-oran/conf/distro/oran-inf.conf index cb09707..6cc1f28 100644 --- a/meta-oran/conf/distro/oran-inf.conf +++ b/meta-oran/conf/distro/oran-inf.conf @@ -30,7 +30,7 @@ DISTRO_NAME = "O-RAN-INF" # Set software versions PREFERRED_VERSION_dpdk ?= "17%" -PREFERRED_VERSION_kubernetes ?= "1.15.2%" +PREFERRED_VERSION_kubernetes ?= "1.16.2%" GOVERSION ?= "1.12%" # Set preferred providers diff --git a/meta-oran/recipes-containers/kubernetes-plugins/flannel_0.12.0.bb b/meta-oran/recipes-containers/kubernetes-plugins/flannel_0.12.0.bb new file mode 100644 index 0000000..09ad0de --- /dev/null +++ b/meta-oran/recipes-containers/kubernetes-plugins/flannel_0.12.0.bb @@ -0,0 +1,35 @@ +# +# Copyright (C) 2019 Wind River Systems, Inc. +# + +SUMMARY = "Flannel is a simple and easy way to configure a layer 3 network fabric designed for Kubernetes." +DESCRIPTION = "\ + Flannel runs a small, single binary agent called flanneld on each host, \ + and is responsible for allocating a subnet lease to each host out of a \ + larger, preconfigured address space. Flannel uses either the Kubernetes \ + API or etcd directly to store the network configuration, the allocated \ + subnets, and any auxiliary data (such as the host's public IP). Packets \ + are forwarded using one of several backend mechanisms including VXLAN and \ + various cloud integrations. \ +" +HOMEPAGE = "https://github.com/coreos/flannel" + +LICENSE = "Apache-2.0" +LIC_FILES_CHKSUM = "file://LICENSE;md5=3b83ef96387f14655fc854ddc3c6bd57" + +SRC_URI = "\ + https://github.com/coreos/flannel/archive/v${PV}.tar.gz;downloadfilename=${BPN}-v${PV}.tar.gz \ +" + +SRC_URI[md5sum] = "1007747571bc6b8c951f72f64e567205" +SRC_URI[sha256sum] = "7375318b288bcff733aabfe1a1007d478cb9091cdaffe68c8253ddd93bc070ed" + +S = "${WORKDIR}/${BPN}-${PV}" + +K8S_PLUGINS = "${sysconfdir}/kubernetes/plugins/${BPN}" + +do_install() { + install -d ${D}${K8S_PLUGINS} + install -m 644 ${S}/README.md ${D}${K8S_PLUGINS} + install -m 644 ${S}/Documentation/kube-flannel.yml ${D}${K8S_PLUGINS} +} diff --git a/meta-oran/recipes-containers/kubernetes-plugins/kubernetes-dashboard_1.10.1.bb b/meta-oran/recipes-containers/kubernetes-plugins/kubernetes-dashboard_1.10.1.bb new file mode 100644 index 0000000..2ac4843 --- /dev/null +++ b/meta-oran/recipes-containers/kubernetes-plugins/kubernetes-dashboard_1.10.1.bb @@ -0,0 +1,35 @@ +# +# Copyright (C) 2019 Wind River Systems, Inc. +# + +SUMMARY = "General-purpose web UI for Kubernetes clusters" +DESCRIPTION = "\ + Kubernetes Dashboard is a general purpose, web-based UI \ + for Kubernetes clusters. It allows users to manage applications \ + running in the cluster and troubleshoot them, as well as manage \ + the cluster itself. \ +" +HOMEPAGE = "https://github.com/kubernetes/dashboard" + +LICENSE = "Apache-2.0" +LIC_FILES_CHKSUM = "file://LICENSE;md5=b1e01b26bacfc2232046c90a330332b3" + +SRC_URI = "\ + https://github.com/kubernetes/dashboard/archive/v${PV}.tar.gz;downloadfilename=${BPN}-v${PV}.tar.gz \ + file://kubernetes-dashboard-admin.rbac.yaml \ + file://kubernetes-dashboard.yaml-set-the-NodePort-type.patch \ +" + +SRC_URI[md5sum] = "52c30b2c4e9cfc31a1ea67144e81513f" +SRC_URI[sha256sum] = "088ba30335e2effec4c5386e97ef25f47f9d042b99bbdeba9a1e3faf87ad2a6d" + +S = "${WORKDIR}/dashboard-${PV}" + +K8S_PLUGINS = "${sysconfdir}/kubernetes/plugins/${BPN}" + +do_install() { + install -d ${D}${K8S_PLUGINS} + install -m 644 ${WORKDIR}/kubernetes-dashboard-admin.rbac.yaml ${D}${K8S_PLUGINS} + install -m 644 ${S}/src/deploy/recommended/kubernetes-dashboard.yaml ${D}${K8S_PLUGINS} + install -m 644 ${S}/README.md ${D}${K8S_PLUGINS} +} diff --git a/meta-oran/recipes-containers/kubernetes-plugins/multus-cni_3.3.bb b/meta-oran/recipes-containers/kubernetes-plugins/multus-cni_3.3.bb index f6b6dba..554258c 100644 --- a/meta-oran/recipes-containers/kubernetes-plugins/multus-cni_3.3.bb +++ b/meta-oran/recipes-containers/kubernetes-plugins/multus-cni_3.3.bb @@ -32,5 +32,6 @@ do_install() { install -m 644 ${S}/README.md ${D}${K8S_PLUGINS} install -m 644 ${S}/images/entrypoint.sh ${D}${K8S_PLUGINS} install -m 644 ${S}/images/README.md ${D}${K8S_PLUGINS}/README-deployment.md - install -m 644 ${S}/images/multus-daemonset-pre-1.16.yml ${D}${K8S_PLUGINS}/multus-daemonset.yml + install -m 644 ${S}/images/multus-daemonset-pre-1.16.yml ${D}${K8S_PLUGINS}/multus-daemonset-pre-1.16.yml + install -m 644 ${S}/images/multus-daemonset.yml ${D}${K8S_PLUGINS}/multus-daemonset.yml } diff --git a/meta-oran/recipes-containers/kubernetes-plugins/node-feature-discovery_0.5.0.bb b/meta-oran/recipes-containers/kubernetes-plugins/node-feature-discovery_0.5.0.bb new file mode 100644 index 0000000..13f638d --- /dev/null +++ b/meta-oran/recipes-containers/kubernetes-plugins/node-feature-discovery_0.5.0.bb @@ -0,0 +1,49 @@ +# +# Copyright (C) 2019 Wind River Systems, Inc. +# + +SUMMARY = "Node feature discovery for Kubernetes" +DESCRIPTION = "\ + This software enables node feature discovery for Kubernetes. \ + It detects hardware features available on each node in a Kubernetes \ + cluster, and advertises those features using node labels. \ + \ + NFD consists of two software components: \ + - nfd-master is responsible for labeling Kubernetes node objects \ + - nfd-worker is detects features and communicates them to nfd-master. \ + One instance of nfd-worker is supposed to be run on each node of the \ + cluster \ +" +HOMEPAGE = "https://github.com/kubernetes-sigs/node-feature-discovery" + +LICENSE = "Apache-2.0" +LIC_FILES_CHKSUM = "file://LICENSE;md5=e23fadd6ceef8c618fc1c65191d846fa" + +SRC_URI = "\ + https://github.com/kubernetes-sigs/${BPN}/archive/v${PV}.tar.gz;downloadfilename=${BPN}-v${PV}.tar.gz \ +" + +SRC_URI[md5sum] = "8130b178e2d5f5aadcec95210b2882d5" +SRC_URI[sha256sum] = "f351d69e3dbc0e8babe4365e0b5a766cf69e566f89c0191e686f653e17e50b6d" + +S = "${WORKDIR}/${BPN}-${PV}" + +K8S_PLUGINS = "${sysconfdir}/kubernetes/plugins/${BPN}" + +do_configure() { + : +} + +do_compile() { + : +} + +do_install() { + install -d ${D}${K8S_PLUGINS} + install -m 644 ${S}/README.md ${D}${K8S_PLUGINS} + install -m 644 ${S}/nfd-daemonset-combined.yaml.template ${D}${K8S_PLUGINS} + install -m 644 ${S}/nfd-worker.conf.example ${D}${K8S_PLUGINS} + install -m 644 ${S}/nfd-worker-job.yaml.template ${D}${K8S_PLUGINS} + install -m 644 ${S}/nfd-master.yaml.template ${D}${K8S_PLUGINS}/nfd-master.yaml + install -m 644 ${S}/nfd-worker-daemonset.yaml.template ${D}${K8S_PLUGINS}/nfd-worker-daemonset.yaml +} diff --git a/meta-oran/recipes-containers/kubernetes/kubernetes/kubeadm.conf b/meta-oran/recipes-containers/kubernetes/kubernetes/kubeadm.conf new file mode 100644 index 0000000..2fb25ba --- /dev/null +++ b/meta-oran/recipes-containers/kubernetes/kubernetes/kubeadm.conf @@ -0,0 +1,17 @@ +# Note: This dropin only works with kubeadm and kubelet v1.11+ +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" +Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" +# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically +EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env +# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use +# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file. +EnvironmentFile=-/etc/sysconfig/kubelet +ExecStart= +ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh +ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;' +ExecStopPost=/bin/rm -f /var/run/kubelet.pid +Restart=always +StartLimitInterval=0 +RestartSec=10 diff --git a/meta-oran/recipes-containers/kubernetes/kubernetes/kubelet-cgroup-setup.sh b/meta-oran/recipes-containers/kubernetes/kubernetes/kubelet-cgroup-setup.sh new file mode 100644 index 0000000..7efb27a --- /dev/null +++ b/meta-oran/recipes-containers/kubernetes/kubernetes/kubelet-cgroup-setup.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# +# Copyright (c) 2019 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# This script does minimal cgroup setup for kubelet. This creates k8s-infra +# cgroup for a minimal set of resource controllers, and configures cpuset +# attributes to span all online cpus and nodes. This will do nothing if +# the k8s-infra cgroup already exists (i.e., assume already configured). +# NOTE: The creation of directories under /sys/fs/cgroup is volatile, and +# does not persist reboots. The cpuset.mems and cpuset.cpus is later updated +# by puppet kubernetes.pp manifest. +# + +# Define minimal path +PATH=/bin:/usr/bin:/usr/local/bin + +# Log info message to /var/log/daemon.log +function LOG { + logger -p daemon.info "$0($$): $@" +} + +# Log error message to /var/log/daemon.log +function ERROR { + logger -s -p daemon.error "$0($$): ERROR: $@" +} + +# Create minimal cgroup directories and configure cpuset attributes +# pids should be first in the list, since it appears to get auto deleted +function create_cgroup { + local cg_name=$1 + local cg_nodeset=$2 + local cg_cpuset=$3 + + local CGROUP=/sys/fs/cgroup + local CONTROLLERS=("pids" "cpuset" "memory" "cpu,cpuacct" "systemd") + local cnt='' + local CGDIR='' + local RC=0 + + # Create the cgroup for required controllers + for cnt in ${CONTROLLERS[@]}; do + CGDIR=${CGROUP}/${cnt}/${cg_name} + if [ -d ${CGDIR} ]; then + LOG "Nothing to do, already configured: ${CGDIR}." + exit ${RC} + fi + LOG "Creating: ${CGDIR}" + mkdir -p ${CGDIR} + RC=$? + if [ ${RC} -ne 0 ]; then + ERROR "Creating: ${CGDIR}, rc=${RC}" + exit ${RC} + fi + done + + # Customize cpuset attributes + LOG "Configuring cgroup: ${cg_name}, nodeset: ${cg_nodeset}, cpuset: ${cg_cpuset}" + CGDIR=${CGROUP}/cpuset/${cg_name} + local CGMEMS=${CGDIR}/cpuset.mems + local CGCPUS=${CGDIR}/cpuset.cpus + local CGTASKS=${CGDIR}/tasks + + # Assign cgroup memory nodeset + LOG "Assign nodeset ${cg_nodeset} to ${CGMEMS}" + /bin/echo ${cg_nodeset} > ${CGMEMS} + RC=$? + if [ ${RC} -ne 0 ]; then + ERROR "Unable to write to: ${CGMEMS}, rc=${RC}" + exit ${RC} + fi + + # Assign cgroup cpus + LOG "Assign cpuset ${cg_cpuset} to ${CGCPUS}" + /bin/echo ${cg_cpuset} > ${CGCPUS} + RC=$? + if [ ${RC} -ne 0 ]; then + ERROR "Assigning: ${cg_cpuset} to ${CGCPUS}, rc=${RC}" + exit ${RC} + fi + + # Set file ownership + chown root:root ${CGMEMS} ${CGCPUS} ${CGTASKS} + RC=$? + if [ ${RC} -ne 0 ]; then + ERROR "Setting owner for: ${CGMEMS}, ${CGCPUS}, ${CGTASKS}, rc=${RC}" + exit ${RC} + fi + + # Set file mode permissions + chmod 644 ${CGMEMS} ${CGCPUS} ${CGTASKS} + RC=$? + if [ ${RC} -ne 0 ]; then + ERROR "Setting mode for: ${CGMEMS}, ${CGCPUS}, ${CGTASKS}, rc=${RC}" + exit ${RC} + fi + + return ${RC} +} + +if [ $UID -ne 0 ]; then + ERROR "Require sudo/root." + exit 1 +fi + +# Configure default kubepods cpuset to span all online cpus and nodes. +ONLINE_NODESET=$(/bin/cat /sys/devices/system/node/online) +ONLINE_CPUSET=$(/bin/cat /sys/devices/system/cpu/online) + +# Configure kubelet cgroup to match cgroupRoot. +create_cgroup 'k8s-infra' ${ONLINE_NODESET} ${ONLINE_CPUSET} + +exit $? + diff --git a/meta-oran/recipes-containers/kubernetes/kubernetes/kubernetes-accounting.conf b/meta-oran/recipes-containers/kubernetes/kubernetes/kubernetes-accounting.conf new file mode 100644 index 0000000..1e8065e --- /dev/null +++ b/meta-oran/recipes-containers/kubernetes/kubernetes/kubernetes-accounting.conf @@ -0,0 +1,4 @@ +[Manager] +DefaultCPUAccounting=yes +DefaultMemoryAccounting=yes + diff --git a/meta-oran/recipes-containers/kubernetes/kubernetes_1.16.2.bb b/meta-oran/recipes-containers/kubernetes/kubernetes_1.16.2.bb new file mode 100644 index 0000000..6fce453 --- /dev/null +++ b/meta-oran/recipes-containers/kubernetes/kubernetes_1.16.2.bb @@ -0,0 +1,42 @@ +# +# Copyright (C) 2020 Wind River Systems, Inc. +# + +require kubernetes.inc + +PV = "1.16.2+git${SRCREV_kubernetes}" +SRCREV_kubernetes = "c97fe5036ef3df2967d086711e6c0c405941e14b" +SRC_BRANCH = "release-1.16" + +SRC_URI += "\ + file://kubernetes-accounting.conf \ + file://kubeadm.conf \ + file://kubelet-cgroup-setup.sh \ +" + +INSANE_SKIP_${PN} += "textrel" +INSANE_SKIP_${PN}-misc += "textrel" +INSANE_SKIP_kubelet += "textrel" + +SYSTEMD_AUTO_ENABLE_kubelet = "disable" + +inherit go112 + +do_install_append() { + # Install the sysctl config for k8s + # install -d ${D}${sysconfdir}/sysctl.d/ + # install -m 644 -D ${WORKDIR}/k8s.conf ${D}${sysconfdir}/sysctl.d/ + + # kubeadm: + install -d -m 0755 ${D}/${sysconfdir}/systemd/system/kubelete.service.d + install -m 0644 ${WORKDIR}/kubeadm.conf ${D}/${sysconfdir}/systemd/system/kubelete.service.d + + # kubelete-cgroup-setup.sh + install -d -m 0755 ${D}/${bindir} + install -m 0644 ${WORKDIR}/kubelet-cgroup-setup.sh ${D}/${bindir} + + # enable CPU and Memory accounting + install -d -m 0755 ${D}/${sysconfdir}/systemd/system.conf.d + install -m 0644 ${WORKDIR}/kubernetes-accounting.conf ${D}/${sysconfdir}//systemd/system.conf.d/ +} + diff --git a/meta-oran/templates/feature/hosts-ia/files/other_drivers.cfg b/meta-oran/templates/feature/hosts-ia/files/other_drivers.cfg index 46bcb1d..32fca9a 100644 --- a/meta-oran/templates/feature/hosts-ia/files/other_drivers.cfg +++ b/meta-oran/templates/feature/hosts-ia/files/other_drivers.cfg @@ -566,3 +566,5 @@ CONFIG_LIBIPW=m CONFIG_IWLEGACY=m CONFIG_IWL4965=m CONFIG_IWL3945=m +CONFIG_VFIO=m +CONFIG_VFIO_PCI=m -- 2.16.6