From 1b131151d8b1fae636eb6e92e1bf812d9d80584e Mon Sep 17 00:00:00 2001 From: Alok Bhatt Date: Wed, 21 Oct 2020 14:37:35 +0000 Subject: [PATCH] Adding helm chart & documentation for redis-cluster related study ticket Issue-ID: RIC-109 Signed-off-by: Alok Bhatt Change-Id: I57422f0126a73645333e1360f1faf010053d60d1 --- docs/installation-rediscluster.rst | 57 ++++ helm/redis-cluster/.helmignore | 22 ++ helm/redis-cluster/Chart.yaml | 25 ++ helm/redis-cluster/templates/NOTES.txt | 36 +++ helm/redis-cluster/templates/configMap.yaml | 387 +++++++++++++++++++++++ helm/redis-cluster/templates/deployment.yaml | 56 ++++ helm/redis-cluster/templates/service.yaml | 36 +++ helm/redis-cluster/templates/serviceaccount.yaml | 55 ++++ helm/redis-cluster/templates/statefulset.yaml | 85 +++++ helm/redis-cluster/values.yaml | 53 ++++ 10 files changed, 812 insertions(+) create mode 100644 docs/installation-rediscluster.rst create mode 100644 helm/redis-cluster/.helmignore create mode 100644 helm/redis-cluster/Chart.yaml create mode 100644 helm/redis-cluster/templates/NOTES.txt create mode 100644 helm/redis-cluster/templates/configMap.yaml create mode 100644 helm/redis-cluster/templates/deployment.yaml create mode 100644 helm/redis-cluster/templates/service.yaml create mode 100644 helm/redis-cluster/templates/serviceaccount.yaml create mode 100644 helm/redis-cluster/templates/statefulset.yaml create mode 100644 helm/redis-cluster/values.yaml diff --git a/docs/installation-rediscluster.rst b/docs/installation-rediscluster.rst new file mode 100644 index 0000000..e66635e --- /dev/null +++ b/docs/installation-rediscluster.rst @@ -0,0 +1,57 @@ +################################################################################### +# ============LICENSE_START======================================================= +# +# ================================================================================ +# Copyright (C) 2020 Hcl Technologies Limited. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +################################################################################### + +Important +^^^^^^^^^^ +The redis-cluster currently is NOT part of RIC platform & hence is completely optional. +This piece of document has been created as part of delivery item for below jira ticket +https://jira.o-ran-sc.org/browse/RIC-109 +This ticket is about assessing the feasibility of redis-cluster (with data sharding) +supporting desired pod anti-affinity for high availability as per the ticket. + +Overview +^^^^^^^^^^ +This document describes the environment/conditions used to test the feasibility of Redis +cluster set-up as detailed in the above ticket. Redis Cluster is a distributed implementation +of Redis with high performance goals. More details at https://redis.io/topics/cluster-spec + +Environment Set-Up +^^^^^^^^^^ +The set up was tested with kubernetes v1.19 cluster with + #. Pod topology spread constraint enabled + Reference: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints + #. CEPH as the Cluster Storage Solution. + Reference: https://github.com/rook/rook.github.io/blob/master/docs/rook/v1.4/ceph-filesystem.md + #. Three worker nodes in the kubernet cluster + +Execution +^^^^^^^^^^ +Once environment is set-up, a redis-cluster can be set up using the helm-chart (also provided with +this commit). Once cluster is running, any master/slave of the redis instance pods can be deleted which +will be compensated automatically by new instances + +At this stage the perl utility program (included with helm-chart) can be run. The helm chart installation +output generates the requirement commands to invoke. + +This utility program identifies the missing anti-affinity(as per above ticket) of redis instances required +in a redis-cluster. When executed it communicates to redis nodes to switch roles (e.g. master/slave) +such that the end-state meets the desired anti-affinity. + + diff --git a/helm/redis-cluster/.helmignore b/helm/redis-cluster/.helmignore new file mode 100644 index 0000000..50af031 --- /dev/null +++ b/helm/redis-cluster/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/redis-cluster/Chart.yaml b/helm/redis-cluster/Chart.yaml new file mode 100644 index 0000000..ccc6959 --- /dev/null +++ b/helm/redis-cluster/Chart.yaml @@ -0,0 +1,25 @@ +################################################################################### +# ============LICENSE_START======================================================= +# +# ================================================================================ +# Copyright (C) 2020 Hcl Technologies Limited. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +################################################################################### + +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Redis Cluster Created as per study tickeet requirement of https://jira.o-ran-sc.org/browse/RIC-109 +name: redis-cluster +version: 0.1.0 diff --git a/helm/redis-cluster/templates/NOTES.txt b/helm/redis-cluster/templates/NOTES.txt new file mode 100644 index 0000000..1288614 --- /dev/null +++ b/helm/redis-cluster/templates/NOTES.txt @@ -0,0 +1,36 @@ +1. Important: You must wait to ensure that all {{ .Values.rediscluster.replicaCount }} instances of redis are in "Running" state. You can use below command to watch if all the required instances are ready + +watch "kubectl -n {{ .Release.Namespace }} get po -l app.kubernetes.io/instance={{ .Release.Name }}" + +2. Run below command to create redis-cluster. + +kubectl -n {{ .Release.Namespace }} exec -it {{ .Values.rediscluster.name }}-0 -- sh -c "echo yes | redis-cli --cluster create --cluster-replicas 2 \ +$(kubectl -n {{ .Release.Namespace }} get po \ +{{ .Values.rediscluster.name }}-0 \ +{{ .Values.rediscluster.name }}-1 \ +{{ .Values.rediscluster.name }}-6 \ +{{ .Values.rediscluster.name }}-3 \ +{{ .Values.rediscluster.name }}-2 \ +{{ .Values.rediscluster.name }}-4 \ +{{ .Values.rediscluster.name }}-7 \ +{{ .Values.rediscluster.name }}-8 \ +{{ .Values.rediscluster.name }}-5 \ +-o=jsonpath='{range .items[*]}{.status.podIP}{":6379 "}{end}')" + +3. Once cluster is created, you can use below utility to see the related redis nodes (master and slaves) sets, + along with the k8 worknode details where each is placed. + + PLACENODE_POD=$(kubectl get --no-headers po -l app={{ .Values.assigner.label }} -o=jsonpath='{.items[0].metadata.name}') + kubectl exec -it ${PLACENODE_POD} -- sh /conf/relatenode.sh + +4. If previous step shows the undesired state for pod-antiaffinity use below perl program to make it as per desired state. + It will NOT take any action when the set-up is with desired pod-antiaffinity + kubectl exec -it ${PLACENODE_POD} -- perl /conf/placenode.pl + + +5. Run below commands to delete the helm release and the PVC + a. helm delete --purge {{ .Release.Name }} + b. kubectl delete pvc -l app.kubernetes.io/instance={{ .Release.Name }} -n {{ .Release.Namespace }} + + + diff --git a/helm/redis-cluster/templates/configMap.yaml b/helm/redis-cluster/templates/configMap.yaml new file mode 100644 index 0000000..d7da426 --- /dev/null +++ b/helm/redis-cluster/templates/configMap.yaml @@ -0,0 +1,387 @@ +################################################################################### +# ============LICENSE_START======================================================= +# +# ================================================================================ +# Copyright (C) 2020 Hcl Technologies Limited. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +################################################################################### +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.rediscluster.name }}-cm +data: + update-node.sh: | + #!/bin/sh + REDIS_NODES="/data/nodes.conf" + sed -i -e "/myself/ s/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/${POD_IP}/" ${REDIS_NODES} + exec "$@" + + redis.conf: |+ + cluster-enabled yes + cluster-require-full-coverage no + cluster-node-timeout {{ .Values.rediscluster.nodetimeout }} + cluster-config-file /data/nodes.conf + cluster-migration-barrier 1 + appendonly yes + protected-mode no +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.assigner.name }}-cm +data: + placenode.pl: | + #!/usr/bin/env perl + =head + ============LICENSE_START======================================================= + + ================================================================================ + Copyright (C) 2020 Hcl Technologies Limited. + ================================================================================ + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ============LICENSE_END========================================================= + + + About: + + This script has been developed as part of https://jira.o-ran-sc.org/browse/RIC-360 + This script identifies the missing anti-affinity(as per above ticket) of redis instances + required in a redis-cluster. If there is an undesired anti-affinity this script can be + executed to communicate to redis nodes to switch roles (e.g. master/slave) such that the + end-state meets the desired anti-affinity. + + + Pre-requisites: + + 1) A redis cluster with 3 masters (2 replicas each) deployed on kubernetes 1.18 (or later) + 2) Three available worker nodes for serving redis workloads + 3) kubectl (with access to the k8 cluster) + + =cut + + + my $podRow = { + "podIP" => "", + "podName" => "", + "k8Node" => "", + + "rdNodeRole" => "", + "rdNodeID" => "", + + "rdMasterNodeID" => "", + "slaveIPs" => [] + }; + + # Pod label for redis nodes + my $podLabel = $ENV{'POD_LABEL'}; + + my $podTable = []; + my $k8NodeInfo = []; + + setk8NodesInfo(); + validate(); + + # Master + spreadMastersIfRequired(); + # Slave + my $disparity = getSlaveDisparity(); + spreadSlavesIfRequired(); + + sub validate() { + my @masters = map { $_->{'rdNodeRole'} eq 'master' ? $_ : () } @{$podTable}; + if ( @masters > @{$k8NodeInfo->{allk8Nodes}} ) { + print "Info: Skipping any action as num of master > number of k8 nodes..\n"; + exit; + } + } + + + sub spreadSlavesIfRequired() { + + + # Get node with maximum disparity first + my @disparityMatrix = reverse sort { @{$a} <=> @{$b} } @${disparity}; + #@disparityMatrix = grep defined, @disparityMatrix; + #@disparityMatrix = map { defined $_ ? $_ : () } @disparityMatrix; + + # Get list of slaves to be swapped roles. + my @slaveSwapList = (); + my $maxDisparityPerNode = @{$disparityMatrix[0]}; + + for (my $disparityPass=0; $disparityPass < $maxDisparityPerNode; $disparityPass++) { + for (my $k8NodeIndex=0; $k8NodeIndex <= $#{disparityMatrix}; $k8NodeIndex++) { + #print "$disparityMatrix[$disparityPass] && $disparityMatrix[$k8NodeIndex][$disparityPass]"; + if ( $disparityMatrix[$disparityPass] && $disparityMatrix[$k8NodeIndex][$disparityPass] ) { + push(@slaveSwapList,$disparityMatrix[$k8NodeIndex][$disparityPass]); + } + } + } + if ( ! @slaveSwapList ) { + print "Info: No disparity found with slaves.\n" if ( @slaveSwapList < 2); + exit; + } elsif ( @slaveSwapList == 1 ) { + print "Info: single host scenario (with no swap candidate in other k8 nodes) found.\n"; + exit; + } else { + print "Info: slave disparity found.\n"; + } + + # Swap slaves + for (my $swapIndex=0; $swapIndex < @slaveSwapList; $swapIndex++) { + $pod1 = $slaveSwapList[$swapIndex]; + $pod2 = $slaveSwapList[++$swapIndex]; + #print "Info: Swapping Slaves: " . join($pod1->{podName}, $pod2->{podName}) . "\n"; + + my $cmd1 = qq[kubectl exec -it ]. + qq[$pod1->{podName} -- redis-cli -p 6379 cluster replicate $pod2->{rdMasterNodeID} ]; + + my $cmd2 = qq[kubectl exec -it ]. + qq[$pod2->{podName} -- redis-cli -p 6379 cluster replicate $pod1->{rdMasterNodeID} ]; + + runRediClusterCmd($cmd1); + runRediClusterCmd($cmd2); + #print "\n$cmd1"; + #print "\n$cmd2\n"; + } + + } + + + sub getSlaveDisparity() { + + # Get Slave Disparity Metrix + my $disparity = (); + my $nodeIndex = 0; + foreach my $k8NodeName ( @{$k8NodeInfo->{allk8Nodes}} ) { + my @redisNodesOnk8Node = map { $_->{'k8Node'} eq $k8NodeName ? $_ : () } @{$podTable}; + @redisNodesOnk8Node = sort { $a->{"rdNodeRole"} cmp $b->{"rdNodeRole"} } @redisNodesOnk8Node; + + my $master = shift @redisNodesOnk8Node; + + for (my $index=0; $index <= $#{redisNodesOnk8Node}; $index++ ) { + my $slave = $redisNodesOnk8Node[$index]; + #print "chekcing for pod: $slave->{podName}\n"; + my $disparityFound = 0; + if ( $slave->{rdMasterNodeID} eq $master->{rdNodeID} ) { + $disparityFound = 1; + } else { + #check is other slaves are its sibling + for (my $nextIndex=$index + 1; $nextIndex <= $#{redisNodesOnk8Node}; $nextIndex++ ) { + if ( $slave->{rdMasterNodeID} eq $redisNodesOnk8Node[$nextIndex]->{rdMasterNodeID} ) { + $disparityFound = 1; + break; + } + } + } + if ($disparityFound) { + #$disparity[$nodeIndex][$index] = { 'podName' => $slave->{"podName"}, 'rdMasterNodeID' => $slave->{"rdMasterNodeID"} } ; + push(@{$disparity[$nodeIndex]},{ 'podName' => $slave->{"podName"}, 'rdMasterNodeID' => $slave->{"rdMasterNodeID"} } ) ; + } + } + $nodeIndex++; + } + return \@disparity; + } + + sub spreadMastersIfRequired() { + + NODE_WITH_NO_MASTER: foreach my $nodeWithoutMaster (@{$k8NodeInfo->{k8NodesWithoutMaster}}) { + # For each k8Node without any master + # Check for each extra master on its hostNode + # Find its slave on the this hostNode (i.e. without any master) + # Such slave must be Found for 3x3 set-up: + # Then Promote as master # Re-Evaluate + + # Get All Redis Slaves on This k8 node + print "Info: K8 node without any master : $nodeWithoutMaster\n"; + my @rdSlaveNodes = map { ($_->{'k8Node'} eq $nodeWithoutMaster ) && ($_->{'rdNodeRole'} eq 'slave') ? $_ : () } @{$podTable}; + + foreach my $nodeWithExtraMaster (@{$k8NodeInfo->{k8NodesWithExtraMaster}} ) { + print "Info: k8 Node with extra master : $nodeWithExtraMaster\n"; + #my @rdSlaveNodes = map { ($_->{'k8Node'} eq $nodeWithoutMaster ) && ($_->{'rdNodeRole'} eq 'slave') ? $_ : () } @{$podTable}; + + my @masterInstances = map { ($_->{'k8Node'} eq $nodeWithExtraMaster ) && ($_->{'rdNodeRole'} eq 'master') ? $_ : () } @{$podTable}; + foreach my $master (@masterInstances) { + my @slave = map { $_->{"rdMasterNodeID"} eq $master->{rdNodeID} ? $_ : () } @rdSlaveNodes; + if ( @slave ) { + promoteSlaveAsMaster($slave[0]); + my $isPromoted = 0; + my $slaveNodeID= $slave[0]->{rdNodeID}; + while( ! $isPromoted ) { + sleep(8); + setk8NodesInfo(); + my ($promotedNode) = map { $slaveNodeID eq $_->{rdNodeID} ? $_ : () } @{$podTable}; + + if ( $promotedNode->{'rdNodeRole'} ne 'master' ) { + print ("Info: Waiting for node promotion confirmation..\n"); + } else { + $isPromoted = 1; + print ("Info: Node promotion confirmed.\n"); + } + } + next NODE_WITH_NO_MASTER; + } + } + } + } + print "Info: All redis masters are on separate k8 Nodes. \n" if ( ! @{$k8NodeInfo->{k8NodesWithoutMaster}}) ; + } + + sub promoteSlaveAsMaster() { + my $slavePod = shift; + #print "Info: Promoting Slave $slavePod->{'podName'} On $slavePod->{'k8Node'} as master"; + my $cmd = qq[kubectl exec -it $slavePod->{'podName'} -- redis-cli -p 6379 cluster failover takeover]; + runRediClusterCmd($cmd); + + } + sub runRediClusterCmd() { + my $cmd = shift; + print "Info: Running Cmd:$cmd \n"; + `$cmd;`; + sleep(8); + } + + + #foreach my $item (@{$podTable}) { + #} + + # find_nodes_without-a-single_master + sub setk8NodesInfo() { + + $podTable = []; + $k8NodeInfo = []; + + getCurrentStatus(); + # All k8 nodes + my @k8NodeList = uniq(map { $_->{'k8Node'} } @$podTable); + + # Find Nodes with At least One master + my @k8NodesWithMaster; + foreach my $nodeName (@k8NodeList) { + push(@k8NodesWithMaster, map { ($_->{'k8Node'} eq $nodeName) && ($_->{'rdNodeRole'} eq 'master') ? $nodeName : () } @{$podTable} ); + } + + # Find Nodes without any master = All nodes - Nodes with at least one Master + my %k8NodesMap = (); + foreach (@k8NodesWithMaster) { + if ( exists $k8NodesMap{$_} ) { + $k8NodesMap{$_}++; + } else { + $k8NodesMap{$_} = 1; + } + } + my @k8NodesWithoutMaster = map { exists $k8NodesMap{$_} ? () : $_ } @k8NodeList; + my @k8NodesWithExtraMaster = uniq(map { $k8NodesMap{$_} > 1 ? $_ : () } @k8NodesWithMaster); + + $k8NodeInfo = { 'allk8Nodes' => \@k8NodeList, 'k8NodesWithExtraMaster' => \@k8NodesWithExtraMaster, 'k8NodesWithoutMaster' => \@k8NodesWithoutMaster }; + } + + + + + + # Validate if number of masters ,= number of rea + + # + #sub filter + + =head + get + podName where k8Node eq "x" + get position of k8node eq x + where + =cut + + exit; + + sub uniq { + my %seen; + grep !$seen{$_}++, @_; + } + + sub getCurrentStatus() { + + # Run pod list command + my @getPods = `kubectl get po --no-headers -o wide -l $podLabel |grep Running`; chomp @getPods; + #my @getPods = `kubectl get po --no-headers -o wide -l managed-by=redis-cluster-operator|grep Running`; chomp @getPods; + + foreach my $podLine (@getPods) { + my @podData = split(/\s+/,$podLine); + my ($podName,$status,$age,$podIP,$podNode) = ($podData[0], $podData[2], $podData[4], $podData[5],$podData[6]); + + #print "$podName,$status,$age,$podIP,$podNode" ."\n"; + my $podRow = { 'podIP' => $podIP, 'podName' => $podName, 'k8Node' => $podNode, 'podAge' => $age, 'podStatus' => $status }; + push (@{$podTable},$podRow) + } + + my $podName = $podTable->[0]{'podName'}; + #print "Info:kubectl exec $podName -- cat nodes.conf|sort -k3\n"; + my @rdNodeData = `kubectl exec $podName -- cat nodes.conf|sort -k3`; chomp @rdNodeData; + foreach my $rdNodeLine (@rdNodeData) { + next if ($rdNodeLine !~ /master|slave/); + my @rdNodeData = split(/\s+/,$rdNodeLine); + my ($rdNodeID,$rdRole,$rdMasterNodeID,$epoch) = ($rdNodeData[0], $rdNodeData[2], $rdNodeData[3],$rdNodeData[5]); + my ($podIP) = split(/:/,$rdNodeData[1]); + $rdRole =~ s/myself,//; + + #print "$rdNodeID,$rdRole,$rdMasterNodeID,$podIP" ."\n"; + my $rdElem = { 'podIP' => $podIP, + 'rdNodeID' => $rdNodeID, + 'rdRole' => $rdRole, + 'rdMasterNodeID' => $rdMasterNodeID, + 'epoch' => $epoch + }; + + for(my $index=0; $index <= $#{$podTable}; $index++) { + if ( $podTable->[$index]{'podIP'} eq $podIP ) { + #print "Matched\n"; + $podTable->[$index]{'rdNodeID'} = $rdNodeID; + $podTable->[$index]{'rdNodeRole'} = $rdRole; + $podTable->[$index]{'rdMasterNodeID'} = $rdMasterNodeID; + $podTable->[$index]{'epoch'} = $epoch; + } + } + #exit; + + } + } + + relatenode.sh: | + #!/bin/sh + podLabel=${POD_LABEL} + firstPod=$(kubectl get po -o wide -l app.kubernetes.io/name=redis-cluster --no-headers=true|head -1|cut -d" " -f1) + + kubectl get po -o wide -l $podLabel |tail +2|awk '{printf("%s:%s:%s:%s\n",$6,$1,$7,$10)}'|sort > /tmp/1.txt + kubectl exec $firstPod -- cat nodes.conf|sed 's/myself,//'|awk '/master|slave/ {print $2,$1,$3,$4}'|sort > /tmp/2.txt + join -t ":" /tmp/1.txt /tmp/2.txt |sort -k3,4 | sed 's/ /:/g'|awk -F":" '{print $2,$7,$3,$1,$4,$6,$8}' > /tmp/3.txt + + echo "\n POD_NAME ROLE k8NODE POD_IP REDIS_NODE_ID REDIS_MASTER_NODE_ID" + grep $(cut -d" " -f4 /tmp/2.txt|sort -u|grep -v "-"|sed -n '1p') /tmp/3.txt + echo "" + grep $(cut -d" " -f4 /tmp/2.txt|sort -u|grep -v "-"|sed -n '2p') /tmp/3.txt + echo "" + grep $(cut -d" " -f4 /tmp/2.txt|sort -u|grep -v "-"|sed -n '3p') /tmp/3.txt + diff --git a/helm/redis-cluster/templates/deployment.yaml b/helm/redis-cluster/templates/deployment.yaml new file mode 100644 index 0000000..b1526b7 --- /dev/null +++ b/helm/redis-cluster/templates/deployment.yaml @@ -0,0 +1,56 @@ +################################################################################ +# Copyright (c) 2019 AT&T Intellectual Property. # +# Copyright (c) 2019 Nokia. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +################################################################################ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.assigner.name }}-dep + namespace: {{ .Release.Namespace }} + labels: + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Values.assigner.label }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ .Values.assigner.label }} + release: {{ .Release.Name }} + spec: + containers: + - name: kubectl + hostname: {{ .Values.assigner.label }} + image: "bitnami/kubectl:1.18" + command: ["/bin/sh"] + args: ["-c", "sleep 3000"] + env: + - name: "POD_LABEL" + value: app.kubernetes.io/instance={{ .Release.Name }} + + volumeMounts: + - name: conf + mountPath: /conf + readOnly: false + volumes: + - name: conf + configMap: + name: {{ .Values.assigner.name }}-cm + defaultMode: 0755 + serviceAccountName: {{ .Values.assigner.name }}-sa diff --git a/helm/redis-cluster/templates/service.yaml b/helm/redis-cluster/templates/service.yaml new file mode 100644 index 0000000..68c571e --- /dev/null +++ b/helm/redis-cluster/templates/service.yaml @@ -0,0 +1,36 @@ +################################################################################### +# ============LICENSE_START======================================================= +# +# ================================================================================ +# Copyright (C) 2020 Hcl Technologies Limited. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +################################################################################### + +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.rediscluster.name }}-svc +spec: + type: {{ .Values.service.type }} + ports: + - port: 6379 + targetPort: 6379 + name: client + - port: 16379 + targetPort: 16379 + name: gossip + selector: + app.kubernetes.io/name: {{ .Values.rediscluster.name }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/helm/redis-cluster/templates/serviceaccount.yaml b/helm/redis-cluster/templates/serviceaccount.yaml new file mode 100644 index 0000000..b799967 --- /dev/null +++ b/helm/redis-cluster/templates/serviceaccount.yaml @@ -0,0 +1,55 @@ +################################################################################### +# ============LICENSE_START======================================================= +# +# ================================================================================ +# Copyright (C) 2020 Hcl Technologies Limited. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +################################################################################### +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.assigner.name }}-sa + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Values.assigner.name }}-role + +rules: + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Values.assigner.name }}-rb +subjects: + - kind: ServiceAccount + name: {{ .Values.assigner.name }}-sa +roleRef: + kind: Role + name: {{ .Values.assigner.name }}-role + apiGroup: rbac.authorization.k8s.io + diff --git a/helm/redis-cluster/templates/statefulset.yaml b/helm/redis-cluster/templates/statefulset.yaml new file mode 100644 index 0000000..afebbaa --- /dev/null +++ b/helm/redis-cluster/templates/statefulset.yaml @@ -0,0 +1,85 @@ +################################################################################### +# ============LICENSE_START======================================================= +# +# ================================================================================ +# Copyright (C) 2020 Hcl Technologies Limited. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +################################################################################### + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Values.rediscluster.name }} +spec: + serviceName: {{ .Values.service.name }} + replicas: {{ .Values.rediscluster.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ .Values.rediscluster.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ .Values.rediscluster.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: {{ .Values.topology.policyUnstisfiable }} + #whenUnsatisfiable: ScheduleAnyway + #whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: {{ .Values.rediscluster.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + containers: + - name: {{ .Values.container.name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: 6379 + name: client + - containerPort: 16379 + name: gossip + command: ["/conf/update-node.sh", "redis-server", "/conf/redis.conf"] + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: conf + mountPath: /conf + readOnly: false + - name: data + mountPath: /data + readOnly: false + volumes: + - name: conf + configMap: + name: {{ .Values.rediscluster.name }}-cm + defaultMode: 0755 + volumeClaimTemplates: + - metadata: + name: data + labels: + app.kubernetes.io/name: {{ .Values.rediscluster.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.volume.storage }} diff --git a/helm/redis-cluster/values.yaml b/helm/redis-cluster/values.yaml new file mode 100644 index 0000000..39112d8 --- /dev/null +++ b/helm/redis-cluster/values.yaml @@ -0,0 +1,53 @@ +################################################################################### +# ============LICENSE_START======================================================= +# +# ================================================================================ +# Copyright (C) 2020 Hcl Technologies Limited. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +################################################################################### + +################################################################# +# Application configuration defaults. +################################################################# + +rediscluster: + nodetimeout: 15000 + replicaCount: 9 + name: redis-cluster + +image: + repository: redis + tag: 5.0.1-alpine + pullPolicy: IfNotPresent + +container: + name: redis + +topology: + policyUnstisfiable: DoNotSchedule + +nameOverride: "" +fullnameOverride: "" + +service: + type: ClusterIP + +volume: + storage: "1Gi" + +# Assigner is reponsible for assigning appropriate master/slave roles to redis nodes +assigner: + name: assigner + label: assigner -- 2.16.6