1 ###################################################################################
2 # ============LICENSE_START=======================================================
4 # ================================================================================
5 # Copyright (C) 2020 Hcl Technologies Limited.
6 # ================================================================================
7 # Licensed under the Apache License, Version 2.0 (the "License");
8 # you may not use this file except in compliance with the License.
9 # You may obtain a copy of the License at
11 # http://www.apache.org/licenses/LICENSE-2.0
13 # Unless required by applicable law or agreed to in writing, software
14 # distributed under the License is distributed on an "AS IS" BASIS,
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
18 # ============LICENSE_END=========================================================
19 ###################################################################################
24 name: {{ .Values.rediscluster.name }}-cm
28 REDIS_NODES="/data/nodes.conf"
29 sed -i -e "/myself/ s/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/${POD_IP}/" ${REDIS_NODES}
34 cluster-require-full-coverage no
35 cluster-node-timeout {{ .Values.rediscluster.nodetimeout }}
36 cluster-config-file /data/nodes.conf
37 cluster-migration-barrier 1
44 name: {{ .Values.assigner.name }}-cm
49 ============LICENSE_START=======================================================
51 ================================================================================
52 Copyright (C) 2020 Hcl Technologies Limited.
53 ================================================================================
54 Licensed under the Apache License, Version 2.0 (the "License");
55 you may not use this file except in compliance with the License.
56 You may obtain a copy of the License at
58 http://www.apache.org/licenses/LICENSE-2.0
60 Unless required by applicable law or agreed to in writing, software
61 distributed under the License is distributed on an "AS IS" BASIS,
62 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
63 See the License for the specific language governing permissions and
64 limitations under the License.
65 ============LICENSE_END=========================================================
70 This script has been developed as part of https://jira.o-ran-sc.org/browse/RIC-360
71 This script identifies the missing anti-affinity(as per above ticket) of redis instances
72 required in a redis-cluster. If there is an undesired anti-affinity this script can be
73 executed to communicate to redis nodes to switch roles (e.g. master/slave) such that the
74 end-state meets the desired anti-affinity.
79 1) A redis cluster with 3 masters (2 replicas each) deployed on kubernetes 1.18 (or later)
80 2) Three available worker nodes for serving redis workloads
81 3) kubectl (with access to the k8 cluster)
94 "rdMasterNodeID" => "",
98 # Pod label for redis nodes
99 my $podLabel = $ENV{'POD_LABEL'};
108 spreadMastersIfRequired();
110 my $disparity = getSlaveDisparity();
111 spreadSlavesIfRequired();
114 my @masters = map { $_->{'rdNodeRole'} eq 'master' ? $_ : () } @{$podTable};
115 if ( @masters > @{$k8NodeInfo->{allk8Nodes}} ) {
116 print "Info: Skipping any action as num of master > number of k8 nodes..\n";
122 sub spreadSlavesIfRequired() {
125 # Get node with maximum disparity first
126 my @disparityMatrix = reverse sort { @{$a} <=> @{$b} } @${disparity};
127 #@disparityMatrix = grep defined, @disparityMatrix;
128 #@disparityMatrix = map { defined $_ ? $_ : () } @disparityMatrix;
130 # Get list of slaves to be swapped roles.
131 my @slaveSwapList = ();
132 my $maxDisparityPerNode = @{$disparityMatrix[0]};
134 for (my $disparityPass=0; $disparityPass < $maxDisparityPerNode; $disparityPass++) {
135 for (my $k8NodeIndex=0; $k8NodeIndex <= $#{disparityMatrix}; $k8NodeIndex++) {
136 #print "$disparityMatrix[$disparityPass] && $disparityMatrix[$k8NodeIndex][$disparityPass]";
137 if ( $disparityMatrix[$disparityPass] && $disparityMatrix[$k8NodeIndex][$disparityPass] ) {
138 push(@slaveSwapList,$disparityMatrix[$k8NodeIndex][$disparityPass]);
142 if ( ! @slaveSwapList ) {
143 print "Info: No disparity found with slaves.\n" if ( @slaveSwapList < 2);
145 } elsif ( @slaveSwapList == 1 ) {
146 print "Info: single host scenario (with no swap candidate in other k8 nodes) found.\n";
149 print "Info: slave disparity found.\n";
153 for (my $swapIndex=0; $swapIndex < @slaveSwapList; $swapIndex++) {
154 $pod1 = $slaveSwapList[$swapIndex];
155 $pod2 = $slaveSwapList[++$swapIndex];
156 #print "Info: Swapping Slaves: " . join($pod1->{podName}, $pod2->{podName}) . "\n";
158 my $cmd1 = qq[kubectl exec -it ].
159 qq[$pod1->{podName} -- redis-cli -p 6379 cluster replicate $pod2->{rdMasterNodeID} ];
161 my $cmd2 = qq[kubectl exec -it ].
162 qq[$pod2->{podName} -- redis-cli -p 6379 cluster replicate $pod1->{rdMasterNodeID} ];
164 runRediClusterCmd($cmd1);
165 runRediClusterCmd($cmd2);
173 sub getSlaveDisparity() {
175 # Get Slave Disparity Metrix
178 foreach my $k8NodeName ( @{$k8NodeInfo->{allk8Nodes}} ) {
179 my @redisNodesOnk8Node = map { $_->{'k8Node'} eq $k8NodeName ? $_ : () } @{$podTable};
180 @redisNodesOnk8Node = sort { $a->{"rdNodeRole"} cmp $b->{"rdNodeRole"} } @redisNodesOnk8Node;
182 my $master = shift @redisNodesOnk8Node;
184 for (my $index=0; $index <= $#{redisNodesOnk8Node}; $index++ ) {
185 my $slave = $redisNodesOnk8Node[$index];
186 #print "chekcing for pod: $slave->{podName}\n";
187 my $disparityFound = 0;
188 if ( $slave->{rdMasterNodeID} eq $master->{rdNodeID} ) {
191 #check is other slaves are its sibling
192 for (my $nextIndex=$index + 1; $nextIndex <= $#{redisNodesOnk8Node}; $nextIndex++ ) {
193 if ( $slave->{rdMasterNodeID} eq $redisNodesOnk8Node[$nextIndex]->{rdMasterNodeID} ) {
199 if ($disparityFound) {
200 #$disparity[$nodeIndex][$index] = { 'podName' => $slave->{"podName"}, 'rdMasterNodeID' => $slave->{"rdMasterNodeID"} } ;
201 push(@{$disparity[$nodeIndex]},{ 'podName' => $slave->{"podName"}, 'rdMasterNodeID' => $slave->{"rdMasterNodeID"} } ) ;
209 sub spreadMastersIfRequired() {
211 NODE_WITH_NO_MASTER: foreach my $nodeWithoutMaster (@{$k8NodeInfo->{k8NodesWithoutMaster}}) {
212 # For each k8Node without any master
213 # Check for each extra master on its hostNode
214 # Find its slave on the this hostNode (i.e. without any master)
215 # Such slave must be Found for 3x3 set-up:
216 # Then Promote as master # Re-Evaluate
218 # Get All Redis Slaves on This k8 node
219 print "Info: K8 node without any master : $nodeWithoutMaster\n";
220 my @rdSlaveNodes = map { ($_->{'k8Node'} eq $nodeWithoutMaster ) && ($_->{'rdNodeRole'} eq 'slave') ? $_ : () } @{$podTable};
222 foreach my $nodeWithExtraMaster (@{$k8NodeInfo->{k8NodesWithExtraMaster}} ) {
223 print "Info: k8 Node with extra master : $nodeWithExtraMaster\n";
224 #my @rdSlaveNodes = map { ($_->{'k8Node'} eq $nodeWithoutMaster ) && ($_->{'rdNodeRole'} eq 'slave') ? $_ : () } @{$podTable};
226 my @masterInstances = map { ($_->{'k8Node'} eq $nodeWithExtraMaster ) && ($_->{'rdNodeRole'} eq 'master') ? $_ : () } @{$podTable};
227 foreach my $master (@masterInstances) {
228 my @slave = map { $_->{"rdMasterNodeID"} eq $master->{rdNodeID} ? $_ : () } @rdSlaveNodes;
230 promoteSlaveAsMaster($slave[0]);
232 my $slaveNodeID= $slave[0]->{rdNodeID};
233 while( ! $isPromoted ) {
236 my ($promotedNode) = map { $slaveNodeID eq $_->{rdNodeID} ? $_ : () } @{$podTable};
238 if ( $promotedNode->{'rdNodeRole'} ne 'master' ) {
239 print ("Info: Waiting for node promotion confirmation..\n");
242 print ("Info: Node promotion confirmed.\n");
245 next NODE_WITH_NO_MASTER;
250 print "Info: All redis masters are on separate k8 Nodes. \n" if ( ! @{$k8NodeInfo->{k8NodesWithoutMaster}}) ;
253 sub promoteSlaveAsMaster() {
254 my $slavePod = shift;
255 #print "Info: Promoting Slave $slavePod->{'podName'} On $slavePod->{'k8Node'} as master";
256 my $cmd = qq[kubectl exec -it $slavePod->{'podName'} -- redis-cli -p 6379 cluster failover takeover];
257 runRediClusterCmd($cmd);
260 sub runRediClusterCmd() {
262 print "Info: Running Cmd:$cmd \n";
268 #foreach my $item (@{$podTable}) {
271 # find_nodes_without-a-single_master
272 sub setk8NodesInfo() {
279 my @k8NodeList = uniq(map { $_->{'k8Node'} } @$podTable);
281 # Find Nodes with At least One master
282 my @k8NodesWithMaster;
283 foreach my $nodeName (@k8NodeList) {
284 push(@k8NodesWithMaster, map { ($_->{'k8Node'} eq $nodeName) && ($_->{'rdNodeRole'} eq 'master') ? $nodeName : () } @{$podTable} );
287 # Find Nodes without any master = All nodes - Nodes with at least one Master
289 foreach (@k8NodesWithMaster) {
290 if ( exists $k8NodesMap{$_} ) {
296 my @k8NodesWithoutMaster = map { exists $k8NodesMap{$_} ? () : $_ } @k8NodeList;
297 my @k8NodesWithExtraMaster = uniq(map { $k8NodesMap{$_} > 1 ? $_ : () } @k8NodesWithMaster);
299 $k8NodeInfo = { 'allk8Nodes' => \@k8NodeList, 'k8NodesWithExtraMaster' => \@k8NodesWithExtraMaster, 'k8NodesWithoutMaster' => \@k8NodesWithoutMaster };
306 # Validate if number of masters ,= number of rea
313 podName where k8Node eq "x"
314 get position of k8node eq x
322 grep !$seen{$_}++, @_;
325 sub getCurrentStatus() {
327 # Run pod list command
328 my @getPods = `kubectl get po --no-headers -o wide -l $podLabel |grep Running`; chomp @getPods;
329 #my @getPods = `kubectl get po --no-headers -o wide -l managed-by=redis-cluster-operator|grep Running`; chomp @getPods;
331 foreach my $podLine (@getPods) {
332 my @podData = split(/\s+/,$podLine);
333 my ($podName,$status,$age,$podIP,$podNode) = ($podData[0], $podData[2], $podData[4], $podData[5],$podData[6]);
335 #print "$podName,$status,$age,$podIP,$podNode" ."\n";
336 my $podRow = { 'podIP' => $podIP, 'podName' => $podName, 'k8Node' => $podNode, 'podAge' => $age, 'podStatus' => $status };
337 push (@{$podTable},$podRow)
340 my $podName = $podTable->[0]{'podName'};
341 #print "Info:kubectl exec $podName -- cat nodes.conf|sort -k3\n";
342 my @rdNodeData = `kubectl exec $podName -- cat nodes.conf|sort -k3`; chomp @rdNodeData;
343 foreach my $rdNodeLine (@rdNodeData) {
344 next if ($rdNodeLine !~ /master|slave/);
345 my @rdNodeData = split(/\s+/,$rdNodeLine);
346 my ($rdNodeID,$rdRole,$rdMasterNodeID,$epoch) = ($rdNodeData[0], $rdNodeData[2], $rdNodeData[3],$rdNodeData[5]);
347 my ($podIP) = split(/:/,$rdNodeData[1]);
348 $rdRole =~ s/myself,//;
350 #print "$rdNodeID,$rdRole,$rdMasterNodeID,$podIP" ."\n";
351 my $rdElem = { 'podIP' => $podIP,
352 'rdNodeID' => $rdNodeID,
354 'rdMasterNodeID' => $rdMasterNodeID,
358 for(my $index=0; $index <= $#{$podTable}; $index++) {
359 if ( $podTable->[$index]{'podIP'} eq $podIP ) {
361 $podTable->[$index]{'rdNodeID'} = $rdNodeID;
362 $podTable->[$index]{'rdNodeRole'} = $rdRole;
363 $podTable->[$index]{'rdMasterNodeID'} = $rdMasterNodeID;
364 $podTable->[$index]{'epoch'} = $epoch;
374 podLabel=${POD_LABEL}
375 firstPod=$(kubectl get po -o wide -l app.kubernetes.io/name=redis-cluster --no-headers=true|head -1|cut -d" " -f1)
377 kubectl get po -o wide -l $podLabel |tail +2|awk '{printf("%s:%s:%s:%s\n",$6,$1,$7,$10)}'|sort > /tmp/1.txt
378 kubectl exec $firstPod -- cat nodes.conf|sed 's/myself,//'|awk '/master|slave/ {print $2,$1,$3,$4}'|sort > /tmp/2.txt
379 join -t ":" /tmp/1.txt /tmp/2.txt |sort -k3,4 | sed 's/ /:/g'|awk -F":" '{print $2,$7,$3,$1,$4,$6,$8}' > /tmp/3.txt
381 echo "\n POD_NAME ROLE k8NODE POD_IP REDIS_NODE_ID REDIS_MASTER_NODE_ID"
382 grep $(cut -d" " -f4 /tmp/2.txt|sort -u|grep -v "-"|sed -n '1p') /tmp/3.txt
384 grep $(cut -d" " -f4 /tmp/2.txt|sort -u|grep -v "-"|sed -n '2p') /tmp/3.txt
386 grep $(cut -d" " -f4 /tmp/2.txt|sort -u|grep -v "-"|sed -n '3p') /tmp/3.txt