Implement SDL CLI 'healthcheck' -command 58/6958/10
authorTimo Tietavainen <timo.tietavainen@nokia.com>
Tue, 26 Oct 2021 06:11:53 +0000 (09:11 +0300)
committerTimo Tietavainen <timo.tietavainen@nokia.com>
Mon, 8 Nov 2021 05:27:38 +0000 (07:27 +0200)
Implement a new 'healthcheck' -command for SDL CLI to validate
healthiness of the SDL database. Under the hood 'healthcheck' -command
calls Redis client's 'Master' and 'Slaves' APIs and parses from the
responses master and slave Redis servers related fields to show the
overall healthiness of the SDL database backend.

Redis client's version was upgraded from v6.15.9 to v7 v7.4.1,
because old client didn't support 'Master' and 'Slaves' Redis
sentinel APIs. Also implement usage of Redis client's
'NewSentinelClient' client connection to read state information via
'Master' and 'Slaves' APIs. In case of standalone Redis server
deployment state information is read via Redis client's 'Info' API.

Issue-Id: RIC-113

Signed-off-by: Timo Tietavainen <timo.tietavainen@nokia.com>
Change-Id: I7d964ac5c3d10cc90075977e2b11da7a9c11949d

15 files changed:
go.mod
go.sum
internal/cli/cli_private_fn_test.go [new file with mode: 0644]
internal/cli/healthcheck.go [new file with mode: 0644]
internal/cli/healthcheck_test.go [new file with mode: 0644]
internal/cli/root.go
internal/cli/types.go
internal/cli/utils.go [new file with mode: 0644]
internal/mocks/db_mocks_private_testing.go [new file with mode: 0644]
internal/sdlgoredis/dbinfo.go [new file with mode: 0644]
internal/sdlgoredis/dbstate.go [new file with mode: 0644]
internal/sdlgoredis/dbstate_test.go [new file with mode: 0644]
internal/sdlgoredis/sdlgoredis.go
internal/sdlgoredis/sdlgoredis_test.go
internal/sdlgoredis/sdlgosentinel.go [new file with mode: 0644]

diff --git a/go.mod b/go.mod
index 7f50555..8085e16 100644 (file)
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,7 @@ module gerrit.o-ran-sc.org/r/ric-plt/sdlgo
 go 1.12
 
 require (
-       github.com/go-redis/redis v6.15.9+incompatible
+       github.com/go-redis/redis/v7 v7.4.1
        github.com/onsi/ginkgo v1.14.0 // indirect
        github.com/spf13/cobra v1.1.1
        github.com/stretchr/testify v1.3.0
diff --git a/go.sum b/go.sum
index d5ee04e..fcfcb96 100644 (file)
--- a/go.sum
+++ b/go.sum
@@ -45,8 +45,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
-github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOrTI=
+github.com/go-redis/redis/v7 v7.4.1/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -140,9 +140,11 @@ github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
@@ -235,6 +237,7 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -261,6 +264,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4=
@@ -321,8 +325,9 @@ google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyz
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
diff --git a/internal/cli/cli_private_fn_test.go b/internal/cli/cli_private_fn_test.go
new file mode 100644 (file)
index 0000000..831b5f9
--- /dev/null
@@ -0,0 +1,32 @@
+/*
+   Copyright (c) 2021 AT&T Intellectual Property.
+   Copyright (c) 2018-2021 Nokia.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+ * This source code is part of the near-RT RIC (RAN Intelligent Controller)
+ * platform project (RICP).
+ */
+
+package cli
+
+import (
+       "github.com/spf13/cobra"
+)
+
+// NewHealthCheckCmdForTest is used only in unit tests to mock database.
+func NewHealthCheckCmdForTest(dbCreateCb DbCreateCb) *cobra.Command {
+       return newHealthCheckCmd(dbCreateCb)
+}
diff --git a/internal/cli/healthcheck.go b/internal/cli/healthcheck.go
new file mode 100644 (file)
index 0000000..94467a9
--- /dev/null
@@ -0,0 +1,106 @@
+/*
+   Copyright (c) 2021 AT&T Intellectual Property.
+   Copyright (c) 2018-2021 Nokia.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+ * This source code is part of the near-RT RIC (RAN Intelligent Controller)
+ * platform project (RICP).
+ */
+
+package cli
+
+import (
+       "bytes"
+       "fmt"
+       "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/sdlgoredis"
+       "github.com/spf13/cobra"
+       "os"
+)
+
+func NewHealthCheckCmd() *cobra.Command {
+       return newHealthCheckCmd(newDatabase)
+}
+
+func newHealthCheckCmd(dbCreateCb DbCreateCb) *cobra.Command {
+       cmd := &cobra.Command{
+               Use:   "healthcheck",
+               Short: "healthcheck - validates database healthiness",
+               Long:  `healthcheck - validates database healthiness`,
+               RunE: func(cmd *cobra.Command, args []string) error {
+                       var buf bytes.Buffer
+                       sdlgoredis.SetDbLogger(&buf)
+                       out, err := runHealthCheck(dbCreateCb)
+                       cmd.Println(out)
+                       if err != nil {
+                               cmd.PrintErrf("%s\n", buf.String())
+                       }
+                       return err
+               },
+       }
+       cmd.SetOut(os.Stdout)
+       return cmd
+}
+
+func runHealthCheck(dbCreateCb DbCreateCb) (string, error) {
+       var anyErr error
+       var str string
+       var states []sdlgoredis.DbState
+       for _, dbInst := range dbCreateCb().Instances {
+               info, err := dbInst.State()
+               if err != nil {
+                       anyErr = fmt.Errorf("SDL CLI error: %v", err)
+               }
+               states = append(states, *info)
+       }
+       str = writeStateResults(states)
+       return str, anyErr
+}
+
+func writeStateResults(dbStates []sdlgoredis.DbState) string {
+       var str string
+       var anyErr error
+       for i, dbState := range dbStates {
+               if err := dbState.IsOnline(); err != nil {
+                       anyErr = err
+               }
+               str = str + fmt.Sprintf("  SDL DB backend #%d\n", (i+1))
+               mAddr := dbState.MasterDbState.GetAddress()
+               err := dbState.MasterDbState.IsOnline()
+               if err == nil {
+                       str = str + fmt.Sprintf("    Master (%s): OK\n", mAddr)
+               } else {
+                       str = str + fmt.Sprintf("    Master (%s): NOK\n", mAddr)
+                       str = str + fmt.Sprintf("      %s\n", err.Error())
+               }
+               if dbState.ReplicasDbState != nil {
+                       for j, rInfo := range dbState.ReplicasDbState.States {
+                               err := rInfo.IsOnline()
+                               if err == nil {
+                                       str = str + fmt.Sprintf("    Replica #%d (%s): OK\n", (j+1), rInfo.GetAddress())
+                               } else {
+                                       str = str + fmt.Sprintf("    Replica #%d (%s): NOK\n", (j+1), rInfo.GetAddress())
+                                       str = str + fmt.Sprintf("      %s\n", err.Error())
+                               }
+                       }
+               }
+       }
+       if anyErr == nil {
+               str = fmt.Sprintf("Overall status: OK\n\n") + str
+       } else {
+               str = fmt.Sprintf("Overall status: NOK\n\n") + str
+       }
+       return str
+}
diff --git a/internal/cli/healthcheck_test.go b/internal/cli/healthcheck_test.go
new file mode 100644 (file)
index 0000000..be98aed
--- /dev/null
@@ -0,0 +1,188 @@
+/*
+   Copyright (c) 2021 AT&T Intellectual Property.
+   Copyright (c) 2018-2021 Nokia.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+ * This source code is part of the near-RT RIC (RAN Intelligent Controller)
+ * platform project (RICP).
+ */
+
+package cli_test
+
+import (
+       "bytes"
+       "errors"
+       "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/cli"
+       "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/mocks"
+       "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/sdlgoredis"
+       "github.com/stretchr/testify/assert"
+       "testing"
+)
+
+var hcMocks *healthCheckMocks
+
+type healthCheckMocks struct {
+       dbIface *mocks.MockDB
+       dbErr   error
+       dbState sdlgoredis.DbState
+}
+
+func setupHcMockMasterDb(ip, port string, replicas uint32) {
+       hcMocks = new(healthCheckMocks)
+       hcMocks.dbState.MasterDbState.Fields.Role = "master"
+       hcMocks.dbState.MasterDbState.Fields.Ip = ip
+       hcMocks.dbState.MasterDbState.Fields.Port = port
+       hcMocks.dbState.MasterDbState.Fields.Flags = "master"
+}
+
+func setupHcMockReplicaDb() {
+       hcMocks = new(healthCheckMocks)
+       hcMocks.dbState.ReplicasDbState = new(sdlgoredis.ReplicasDbState)
+       hcMocks.dbState.ReplicasDbState.States = []*sdlgoredis.ReplicaDbState{
+               &sdlgoredis.ReplicaDbState{
+                       Fields: sdlgoredis.ReplicaDbStateFields{
+                               Role: "slave",
+                       },
+               },
+       }
+}
+
+func addHcMockReplicaDbState(ip, port, masterLinkOk string) {
+       if hcMocks.dbState.ReplicasDbState == nil {
+               hcMocks.dbState.ReplicasDbState = new(sdlgoredis.ReplicasDbState)
+       }
+       hcMocks.dbState.ReplicasDbState.States = append(hcMocks.dbState.ReplicasDbState.States,
+               &sdlgoredis.ReplicaDbState{
+                       Fields: sdlgoredis.ReplicaDbStateFields{
+                               Role:             "slave",
+                               Ip:               ip,
+                               Port:             port,
+                               MasterLinkStatus: masterLinkOk,
+                               Flags:            "slave",
+                       },
+               },
+       )
+}
+
+func newMockDatabase() *cli.Database {
+       db := &cli.Database{}
+       hcMocks.dbIface = new(mocks.MockDB)
+       hcMocks.dbIface.On("State").Return(&hcMocks.dbState, hcMocks.dbErr)
+       db.Instances = append(db.Instances, hcMocks.dbIface)
+       return db
+}
+
+func runHcCli() (string, error) {
+       buf := new(bytes.Buffer)
+       cmd := cli.NewHealthCheckCmdForTest(newMockDatabase)
+       cmd.SetOut(buf)
+
+       err := cmd.Execute()
+
+       return buf.String(), err
+}
+
+func TestCliHealthCheckCanShowHelp(t *testing.T) {
+       var expOkErr error
+       expHelp := "Usage:\n  " + "healthcheck [flags]"
+       expNokErr := errors.New("unknown flag: --some-unknown-flag")
+       tests := []struct {
+               args      string
+               expErr    error
+               expOutput string
+       }{
+               {args: "-h", expErr: expOkErr, expOutput: expHelp},
+               {args: "--help", expErr: expOkErr, expOutput: expHelp},
+               {args: "--some-unknown-flag", expErr: expNokErr, expOutput: expHelp},
+       }
+
+       for _, test := range tests {
+               buf := new(bytes.Buffer)
+               cmd := cli.NewHealthCheckCmd()
+               cmd.SetOut(buf)
+               cmd.SetArgs([]string{test.args})
+
+               err := cmd.Execute()
+
+               stdout := buf.String()
+               assert.Equal(t, test.expErr, err)
+               assert.Contains(t, stdout, test.expOutput)
+       }
+}
+
+func TestCliHealthCheckCanShowHaDeploymentOkStatusCorrectly(t *testing.T) {
+       setupHcMockMasterDb("10.20.30.40", "6379", 2)
+       addHcMockReplicaDbState("1.2.3.4", "6379", "ok")
+       addHcMockReplicaDbState("5.6.7.8", "6379", "ok")
+
+       stdout, err := runHcCli()
+
+       assert.Nil(t, err)
+       assert.Contains(t, stdout, "Overall status: OK")
+       assert.Contains(t, stdout, "Master (10.20.30.40:6379): OK")
+       assert.Contains(t, stdout, "Replica #1 (1.2.3.4:6379): OK")
+       assert.Contains(t, stdout, "Replica #2 (5.6.7.8:6379): OK")
+
+}
+
+func TestCliHealthCheckCanShowHaDeploymentStatusCorrectlyWhenOneReplicaStateNotUp(t *testing.T) {
+       setupHcMockMasterDb("10.20.30.40", "6379", 2)
+       addHcMockReplicaDbState("1.2.3.4", "6379", "ok")
+       addHcMockReplicaDbState("5.6.7.8", "6379", "nok")
+
+       stdout, err := runHcCli()
+
+       assert.Nil(t, err)
+       assert.Contains(t, stdout, "Overall status: NOK")
+       assert.Contains(t, stdout, "Replica #2 (5.6.7.8:6379): NOK")
+       assert.Contains(t, stdout, "Replica link to the master is down")
+}
+
+func TestCliHealthCheckCanShowHaDeploymentStatusCorrectlyWhenDbStateQueryFails(t *testing.T) {
+       setupHcMockMasterDb("10.20.30.40", "6379", 0)
+       hcMocks.dbErr = errors.New("Some error")
+       expCliErr := errors.New("SDL CLI error: Some error")
+
+       buf := new(bytes.Buffer)
+       cmd := cli.NewHealthCheckCmdForTest(newMockDatabase)
+       cmd.SetErr(buf)
+
+       err := cmd.Execute()
+       stderr := buf.String()
+
+       assert.Equal(t, expCliErr, err)
+       assert.Contains(t, stderr, "Error: "+expCliErr.Error())
+}
+
+func TestCliHealthCheckCanShowHaDeploymentOkStatusCorrectlyWhenDbStateIsFromReplicaServer(t *testing.T) {
+       setupHcMockReplicaDb()
+
+       stdout, err := runHcCli()
+
+       assert.Nil(t, err)
+       assert.Contains(t, stdout, "Overall status: NOK")
+       assert.Contains(t, stdout, "Master (): NOK")
+}
+
+func TestCliHealthCheckCanShowStandaloneDeploymentOkStatusCorrectly(t *testing.T) {
+       setupHcMockMasterDb("10.20.30.40", "6379", 0)
+
+       stdout, err := runHcCli()
+
+       assert.Nil(t, err)
+       assert.Contains(t, stdout, "Overall status: OK")
+       assert.Contains(t, stdout, "Master (10.20.30.40:6379): OK")
+}
index ceef6c2..6d20ebc 100644 (file)
@@ -34,5 +34,6 @@ func NewRootCmd() *cobra.Command {
                Run: func(cmd *cobra.Command, args []string) {
                },
        }
+       cmd.AddCommand(NewHealthCheckCmd())
        return cmd
 }
index ad942c8..de99e62 100644 (file)
 
 package cli
 
-//Name of the SDL CLI application
+import "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/sdlgoredis"
+
+//iDatabase is an interface towards database backend, for the time being
+//sdlgoredis.DB implements this interface.
+type iDatabase interface {
+       Info() (*sdlgoredis.DbInfo, error)
+       State() (*sdlgoredis.DbState, error)
+}
+
+//Database struct is a holder for the internal database instances.
+type Database struct {
+       Instances []iDatabase
+}
+
+//DbCreateCb callback function type to create a new database
+type DbCreateCb func() *Database
+
+//SdlCliApp constant defines the name of the SDL CLI application
 const SdlCliApp = "sdlcli"
diff --git a/internal/cli/utils.go b/internal/cli/utils.go
new file mode 100644 (file)
index 0000000..3702050
--- /dev/null
@@ -0,0 +1,35 @@
+/*
+   Copyright (c) 2021 AT&T Intellectual Property.
+   Copyright (c) 2018-2021 Nokia.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+ * This source code is part of the near-RT RIC (RAN Intelligent Controller)
+ * platform project (RICP).
+ */
+
+package cli
+
+import (
+       "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/sdlgoredis"
+)
+
+func newDatabase() *Database {
+       db := &Database{}
+       for _, v := range sdlgoredis.Create() {
+               db.Instances = append(db.Instances, v)
+       }
+       return db
+}
diff --git a/internal/mocks/db_mocks_private_testing.go b/internal/mocks/db_mocks_private_testing.go
new file mode 100644 (file)
index 0000000..7832ee3
--- /dev/null
@@ -0,0 +1,42 @@
+/*
+   Copyright (c) 2021 AT&T Intellectual Property.
+   Copyright (c) 2018-2021 Nokia.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+ * This source code is part of the near-RT RIC (RAN Intelligent Controller)
+ * platform project (RICP).
+ */
+
+package mocks
+
+import (
+       "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/sdlgoredis"
+       "github.com/stretchr/testify/mock"
+)
+
+type MockDB struct {
+       mock.Mock
+}
+
+func (m *MockDB) Info() (*sdlgoredis.DbInfo, error) {
+       a := m.Called()
+       return a.Get(0).(*sdlgoredis.DbInfo), a.Error(1)
+}
+
+func (m *MockDB) State() (*sdlgoredis.DbState, error) {
+       a := m.Called()
+       return a.Get(0).(*sdlgoredis.DbState), a.Error(1)
+}
diff --git a/internal/sdlgoredis/dbinfo.go b/internal/sdlgoredis/dbinfo.go
new file mode 100644 (file)
index 0000000..b8990e0
--- /dev/null
@@ -0,0 +1,36 @@
+/*
+   Copyright (c) 2021 AT&T Intellectual Property.
+   Copyright (c) 2018-2021 Nokia.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+ * This source code is part of the near-RT RIC (RAN Intelligent Controller)
+ * platform project (RICP).
+ */
+
+package sdlgoredis
+
+//DbInfo struct is a holder for DB information, which is received from
+//sdlgoredis 'info' call's output.
+type DbInfo struct {
+       Fields DbInfoFields
+}
+
+//DbInfoFields struct is a holder for fields, which are read from sdlgoredis
+//'info' call's output.
+type DbInfoFields struct {
+       MasterRole          bool
+       ConnectedReplicaCnt uint32
+}
diff --git a/internal/sdlgoredis/dbstate.go b/internal/sdlgoredis/dbstate.go
new file mode 100644 (file)
index 0000000..543f22e
--- /dev/null
@@ -0,0 +1,138 @@
+/*
+   Copyright (c) 2021 AT&T Intellectual Property.
+   Copyright (c) 2018-2021 Nokia.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+ * This source code is part of the near-RT RIC (RAN Intelligent Controller)
+ * platform project (RICP).
+ */
+
+package sdlgoredis
+
+import (
+       "fmt"
+)
+
+//DbState struct is a holder for DB state information, which is received from
+//sdlgoredis sentinel 'Master' and 'Slaves' calls output.
+type DbState struct {
+       MasterDbState   MasterDbState
+       ReplicasDbState *ReplicasDbState
+}
+
+//MasterDbState struct is a holder for master Redis state information.
+type MasterDbState struct {
+       Err    error
+       Fields MasterDbStateFields
+}
+
+//ReplicasDbState struct is a holder for Redis slaves state information.
+type ReplicasDbState struct {
+       Err    error
+       States []*ReplicaDbState
+}
+
+//ReplicaDbState struct is a holder for one Redis slave state information.
+type ReplicaDbState struct {
+       Fields ReplicaDbStateFields
+}
+
+//MasterDbStateFields struct is a holder for master Redis state information
+//fields which are read from sdlgoredis sentinel 'Master' call output.
+type MasterDbStateFields struct {
+       Role  string
+       Ip    string
+       Port  string
+       Flags string
+}
+
+//ReplicaDbStateFields struct is a holder for slave Redis state information
+//fields which are read from sdlgoredis sentinel 'Slaves' call output.
+type ReplicaDbStateFields struct {
+       Role             string
+       Ip               string
+       Port             string
+       MasterLinkStatus string
+       Flags            string
+}
+
+func (dbst *DbState) IsOnline() error {
+       if err := dbst.MasterDbState.IsOnline(); err != nil {
+               return err
+       }
+       if dbst.ReplicasDbState != nil {
+               if err := dbst.ReplicasDbState.IsOnline(); err != nil {
+                       return err
+               }
+       }
+       return nil
+}
+
+func (mdbst *MasterDbState) IsOnline() error {
+       if mdbst.Err != nil {
+               return mdbst.Err
+       }
+       if mdbst.Fields.Role != "master" {
+               return fmt.Errorf("No master DB, current role '%s'", mdbst.Fields.Role)
+       }
+       if mdbst.Fields.Flags != "master" {
+               return fmt.Errorf("Master flags are '%s', expected 'master'", mdbst.Fields.Flags)
+       }
+       return nil
+}
+
+func (mdbst *MasterDbState) GetAddress() string {
+       if mdbst.Fields.Ip != "" || mdbst.Fields.Port != "" {
+               return mdbst.Fields.Ip + ":" + mdbst.Fields.Port
+       } else {
+               return ""
+       }
+}
+
+func (rdbst *ReplicasDbState) IsOnline() error {
+       if rdbst.Err != nil {
+               return rdbst.Err
+       }
+       for _, state := range rdbst.States {
+               if err := state.IsOnline(); err != nil {
+                       return err
+               }
+       }
+       return nil
+}
+
+func (rdbst *ReplicaDbState) IsOnline() error {
+       if rdbst.Fields.Role != "slave" {
+               return fmt.Errorf("Replica role is '%s', expected 'slave'", rdbst.Fields.Role)
+       }
+
+       if rdbst.Fields.MasterLinkStatus != "ok" {
+               return fmt.Errorf("Replica link to the master is down")
+       }
+
+       if rdbst.Fields.Flags != "slave" {
+               return fmt.Errorf("Replica flags are '%s', expected 'slave'", rdbst.Fields.Flags)
+       }
+       return nil
+}
+
+func (rdbst *ReplicaDbState) GetAddress() string {
+       if rdbst.Fields.Ip != "" || rdbst.Fields.Port != "" {
+               return rdbst.Fields.Ip + ":" + rdbst.Fields.Port
+       } else {
+               return ""
+       }
+}
diff --git a/internal/sdlgoredis/dbstate_test.go b/internal/sdlgoredis/dbstate_test.go
new file mode 100644 (file)
index 0000000..0b03afa
--- /dev/null
@@ -0,0 +1,187 @@
+/*
+   Copyright (c) 2021 AT&T Intellectual Property.
+   Copyright (c) 2018-2021 Nokia.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+ * This source code is part of the near-RT RIC (RAN Intelligent Controller)
+ * platform project (RICP).
+ */
+
+package sdlgoredis_test
+
+import (
+       "errors"
+       "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/sdlgoredis"
+       "github.com/stretchr/testify/assert"
+       "testing"
+)
+
+type dbStateMock struct {
+       state sdlgoredis.DbState
+}
+
+func setupDbState() *dbStateMock {
+       return new(dbStateMock)
+}
+
+func (ds *dbStateMock) setMasterError(err error) {
+       ds.state.MasterDbState.Err = err
+}
+
+func (ds *dbStateMock) setMasterFields(role, ip, port, rCnt, flags string) {
+       ds.state.MasterDbState.Fields.Role = role
+       ds.state.MasterDbState.Fields.Ip = ip
+       ds.state.MasterDbState.Fields.Port = port
+       ds.state.MasterDbState.Fields.Flags = flags
+}
+
+func (ds *dbStateMock) setReplicaError(err error) {
+       if ds.state.ReplicasDbState == nil {
+               ds.state.ReplicasDbState = new(sdlgoredis.ReplicasDbState)
+       }
+       ds.state.ReplicasDbState.Err = err
+}
+
+func (ds *dbStateMock) addReplicaFields(role, ip, port, mls, flags string) {
+       if ds.state.ReplicasDbState == nil {
+               ds.state.ReplicasDbState = new(sdlgoredis.ReplicasDbState)
+       }
+       newState := new(sdlgoredis.ReplicaDbState)
+       newState.Fields.Role = role
+       newState.Fields.Ip = ip
+       newState.Fields.Port = port
+       newState.Fields.MasterLinkStatus = mls
+       newState.Fields.Flags = flags
+       ds.state.ReplicasDbState.States = append(ds.state.ReplicasDbState.States, newState)
+}
+
+func TestIsOnlineWhenSingleMasterSuccessfully(t *testing.T) {
+       st := setupDbState()
+       st.setMasterFields("master", "1.2.3.4", "60000", "0", "master")
+       err := st.state.IsOnline()
+       assert.Nil(t, err)
+}
+
+func TestIsOnlineWhenSingleMasterFailureIfErrorHasSet(t *testing.T) {
+       testErr := errors.New("Some error")
+       st := setupDbState()
+       st.setMasterFields("master", "1.2.3.4", "60000", "0", "master")
+       st.setMasterError(testErr)
+       err := st.state.IsOnline()
+       assert.Equal(t, testErr, err)
+}
+
+func TestIsOnlineWhenSingleMasterFailureIfNotMasterRole(t *testing.T) {
+       expErr := errors.New("No master DB, current role 'not-master'")
+       st := setupDbState()
+       st.setMasterFields("not-master", "1.2.3.4", "60000", "0", "master")
+       err := st.state.IsOnline()
+       assert.Equal(t, expErr, err)
+}
+
+func TestIsOnlineWhenSingleMasterFailureIfErrorFlags(t *testing.T) {
+       expErr := errors.New("Master flags are 'any-error,master', expected 'master'")
+       st := setupDbState()
+       st.setMasterFields("master", "1.2.3.4", "60000", "0", "any-error,master")
+       err := st.state.IsOnline()
+       assert.Equal(t, expErr, err)
+}
+
+func TestGetAddressMasterSuccessfully(t *testing.T) {
+       st := setupDbState()
+       st.setMasterFields("master", "1.2.3.4", "60000", "0", "master")
+       addr := st.state.MasterDbState.GetAddress()
+       assert.Equal(t, "1.2.3.4:60000", addr)
+}
+
+func TestGetAddressMasterFailureNoIpPort(t *testing.T) {
+       st := setupDbState()
+       st.setMasterFields("master", "", "", "0", "master")
+       addr := st.state.MasterDbState.GetAddress()
+       assert.Equal(t, "", addr)
+}
+
+func TestIsOnlineWhenMasterAndTwoReplicasSuccessfully(t *testing.T) {
+       st := setupDbState()
+       st.setMasterFields("master", "1.2.3.4", "60000", "2", "master")
+       st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave")
+       st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave")
+       err := st.state.IsOnline()
+       assert.Nil(t, err)
+}
+
+func TestIsOnlineWhenMasterAndTwoReplicasFailureIfErrorHasSet(t *testing.T) {
+       testErr := errors.New("Some error")
+       st := setupDbState()
+       st.setMasterFields("master", "1.2.3.4", "60000", "2", "master")
+       st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave")
+       st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave")
+       st.setReplicaError(testErr)
+       err := st.state.IsOnline()
+       assert.Equal(t, testErr, err)
+}
+
+func TestIsOnlineWhenMasterAndTwoReplicasFailureIfNotSlaveRole(t *testing.T) {
+       expErr := errors.New("Replica role is 'not-slave', expected 'slave'")
+       st := setupDbState()
+       st.setMasterFields("master", "1.2.3.4", "60000", "2", "master")
+       st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave")
+       st.addReplicaFields("not-slave", "6.7.8.10", "3450", "ok", "slave")
+       err := st.state.IsOnline()
+       assert.Equal(t, expErr, err)
+}
+
+func TestIsOnlineWhenMasterAndTwoReplicasFailureIfMasterLinkDown(t *testing.T) {
+       expErr := errors.New("Replica link to the master is down")
+       st := setupDbState()
+       st.setMasterFields("master", "1.2.3.4", "60000", "2", "master")
+       st.addReplicaFields("slave", "6.7.8.9", "1234", "nok", "slave")
+       st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave")
+       err := st.state.IsOnline()
+       assert.Equal(t, expErr, err)
+}
+
+func TestIsOnlineWhenMasterAndTwoReplicasFailureIfErrorFlags(t *testing.T) {
+       expErr := errors.New("Replica flags are 'any-error,slave', expected 'slave'")
+       st := setupDbState()
+       st.setMasterFields("master", "1.2.3.4", "60000", "2", "master")
+       st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave")
+       st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "any-error,slave")
+       err := st.state.IsOnline()
+       assert.Equal(t, expErr, err)
+}
+
+func TestGetAddressReplicasSuccessfully(t *testing.T) {
+       st := setupDbState()
+       st.setMasterFields("master", "1.2.3.4", "60000", "2", "master")
+       st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave")
+       st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave")
+       addr := st.state.ReplicasDbState.States[0].GetAddress()
+       assert.Equal(t, "6.7.8.9:1234", addr)
+       addr = st.state.ReplicasDbState.States[1].GetAddress()
+       assert.Equal(t, "6.7.8.10:3450", addr)
+}
+
+func TestGetAddressReplicasNoIpPort(t *testing.T) {
+       st := setupDbState()
+       st.setMasterFields("master", "1.2.3.4", "60000", "2", "master")
+       st.addReplicaFields("slave", "", "", "ok", "slave")
+       st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave")
+       addr := st.state.ReplicasDbState.States[0].GetAddress()
+       assert.Equal(t, "", addr)
+       addr = st.state.ReplicasDbState.States[1].GetAddress()
+       assert.Equal(t, "6.7.8.10:3450", addr)
+}
index 570dfa1..5f64960 100644 (file)
@@ -24,8 +24,9 @@ package sdlgoredis
 
 import (
        "errors"
-       "fmt"
-       "github.com/go-redis/redis"
+       "github.com/go-redis/redis/v7"
+       "io"
+       "log"
        "os"
        "strconv"
        "strings"
@@ -57,10 +58,13 @@ type Config struct {
 
 type DB struct {
        client       RedisClient
+       sentinel     RedisSentinelCreateCb
        subscribe    SubscribeFn
        redisModules bool
        sCbMap       *sharedCbMap
        ch           intChannels
+       cfg          Config
+       addr         string
 }
 
 type Subscriber interface {
@@ -92,6 +96,18 @@ type RedisClient interface {
        EvalSha(sha1 string, keys []string, args ...interface{}) *redis.Cmd
        ScriptExists(scripts ...string) *redis.BoolSliceCmd
        ScriptLoad(script string) *redis.StringCmd
+       Info(section ...string) *redis.StringCmd
+}
+
+var dbLogger *log.Logger
+
+func init() {
+       dbLogger = log.New(os.Stdout, "database: ", log.LstdFlags|log.Lshortfile)
+       redis.SetLogger(dbLogger)
+}
+
+func SetDbLogger(out io.Writer) {
+       dbLogger.SetOutput(out)
 }
 
 func checkResultAndError(result interface{}, err error) (bool, error) {
@@ -127,9 +143,10 @@ func subscribeNotifications(client RedisClient, channels ...string) Subscriber {
        return client.Subscribe(channels...)
 }
 
-func CreateDB(client RedisClient, subscribe SubscribeFn) *DB {
+func CreateDB(client RedisClient, subscribe SubscribeFn, sentinelCreateCb RedisSentinelCreateCb, cfg Config, sentinelAddr string) *DB {
        db := DB{
                client:       client,
+               sentinel:     sentinelCreateCb,
                subscribe:    subscribe,
                redisModules: true,
                sCbMap:       &sharedCbMap{cbMap: make(map[string]ChannelNotificationCb, 0)},
@@ -138,6 +155,8 @@ func CreateDB(client RedisClient, subscribe SubscribeFn) *DB {
                        removeChannel: make(chan string),
                        exit:          make(chan bool),
                },
+               cfg:  cfg,
+               addr: sentinelAddr,
        }
 
        return &db
@@ -145,7 +164,7 @@ func CreateDB(client RedisClient, subscribe SubscribeFn) *DB {
 
 func Create() []*DB {
        osimpl := osImpl{}
-       return ReadConfigAndCreateDbClients(osimpl, newRedisClient)
+       return ReadConfigAndCreateDbClients(osimpl, newRedisClient, subscribeNotifications, newRedisSentinel)
 }
 
 func readConfig(osI OS) Config {
@@ -173,38 +192,48 @@ func (osImpl) Getenv(key string, defValue string) string {
        return val
 }
 
-func ReadConfigAndCreateDbClients(osI OS, clientCreator RedisClientCreator) []*DB {
+func ReadConfigAndCreateDbClients(osI OS, clientCreator RedisClientCreator,
+       subscribe SubscribeFn,
+       sentinelCreateCb RedisSentinelCreateCb) []*DB {
        cfg := readConfig(osI)
-       return createDbClients(cfg, clientCreator)
+       return createDbClients(cfg, clientCreator, subscribe, sentinelCreateCb)
 }
 
-func createDbClients(cfg Config, clientCreator RedisClientCreator) []*DB {
+func createDbClients(cfg Config, clientCreator RedisClientCreator,
+       subscribe SubscribeFn,
+       sentinelCreateCb RedisSentinelCreateCb) []*DB {
        if cfg.clusterAddrList == "" {
-               return []*DB{createLegacyDbClient(cfg, clientCreator)}
+               return []*DB{createLegacyDbClient(cfg, clientCreator, subscribe, sentinelCreateCb)}
        }
 
        dbs := []*DB{}
 
        addrList := strings.Split(cfg.clusterAddrList, ",")
        for _, addr := range addrList {
-               db := createDbClient(cfg, addr, clientCreator)
+               db := createDbClient(cfg, addr, clientCreator, subscribe, sentinelCreateCb)
                dbs = append(dbs, db)
        }
        return dbs
 }
 
-func createLegacyDbClient(cfg Config, clientCreator RedisClientCreator) *DB {
-       return createDbClient(cfg, cfg.hostname, clientCreator)
+func createLegacyDbClient(cfg Config, clientCreator RedisClientCreator,
+       subscribe SubscribeFn,
+       sentinelCreateCb RedisSentinelCreateCb) *DB {
+       return createDbClient(cfg, cfg.hostname, clientCreator, subscribe, sentinelCreateCb)
 }
 
-func createDbClient(cfg Config, hostName string, clientCreator RedisClientCreator) *DB {
+func createDbClient(cfg Config, hostName string, clientCreator RedisClientCreator,
+       subscribe SubscribeFn,
+       sentinelCreateCb RedisSentinelCreateCb) *DB {
        var client RedisClient
+       var db *DB
        if cfg.sentinelPort == "" {
                client = clientCreator(hostName, cfg.port, "", false)
+               db = CreateDB(client, subscribe, nil, cfg, hostName)
        } else {
                client = clientCreator(hostName, cfg.sentinelPort, cfg.masterName, true)
+               db = CreateDB(client, subscribe, sentinelCreateCb, cfg, hostName)
        }
-       db := CreateDB(client, subscribeNotifications)
        db.CheckCommands()
        return db
 }
@@ -243,7 +272,7 @@ func (db *DB) CheckCommands() {
                        }
                }
        } else {
-               fmt.Println(err)
+               dbLogger.Printf("SDL DB commands checking failure: %s\n", err)
        }
 }
 
@@ -291,7 +320,7 @@ func (db *DB) SubscribeChannelDB(cb func(string, ...string), channelPrefix, even
                                case exit := <-ch.exit:
                                        if exit {
                                                if err := sub.Close(); err != nil {
-                                                       fmt.Println(err)
+                                                       dbLogger.Printf("SDL DB channel closing failure: %s\n", err)
                                                }
                                                return
                                        }
@@ -455,6 +484,66 @@ func (db *DB) PTTL(key string) (time.Duration, error) {
        return result, err
 }
 
+func (db *DB) Info() (*DbInfo, error) {
+       var info DbInfo
+       resultStr, err := db.client.Info("all").Result()
+       result := strings.Split(strings.ReplaceAll(resultStr, "\r\n", "\n"), "\n")
+       readRedisInfoReplyFields(result, &info)
+       return &info, err
+}
+
+func readRedisInfoReplyFields(input []string, info *DbInfo) {
+       for _, line := range input {
+               if idx := strings.Index(line, "role:"); idx != -1 {
+                       roleStr := line[idx+len("role:"):]
+                       if roleStr == "master" {
+                               info.Fields.MasterRole = true
+                       }
+               } else if idx := strings.Index(line, "connected_slaves:"); idx != -1 {
+                       cntStr := line[idx+len("connected_slaves:"):]
+                       if cnt, err := strconv.ParseUint(cntStr, 10, 32); err == nil {
+                               info.Fields.ConnectedReplicaCnt = uint32(cnt)
+                       }
+               }
+       }
+}
+
+func (db *DB) State() (*DbState, error) {
+       if db.cfg.sentinelPort != "" {
+               //Establish connection to Redis sentinel. The reason why connection is done
+               //here instead of time of the SDL instance creation is that for the time being
+               //sentinel connection is needed only here to get state information and
+               //state information is needed only by 'sdlcli' hence it is not time critical
+               //and also we want to avoid opening unnecessary TCP connections towards Redis
+               //sentinel for every SDL instance. Now it is done only when 'sdlcli' is used.
+               sentinelClient := db.sentinel(&db.cfg, db.addr)
+               return sentinelClient.GetDbState()
+       } else {
+               var dbState DbState
+               info, err := db.Info()
+               if err != nil {
+                       return &dbState, err
+               }
+               dbState = fillDbStateFromDbInfo(info)
+               return &dbState, err
+       }
+}
+
+func fillDbStateFromDbInfo(info *DbInfo) DbState {
+       var dbState DbState
+       if info.Fields.MasterRole == true {
+               dbState = DbState{
+                       MasterDbState: MasterDbState{
+                               Fields: MasterDbStateFields{
+                                       Role:  "master",
+                                       Flags: "master",
+                               },
+                       },
+               }
+       }
+       return dbState
+}
+
 var luaRefresh = redis.NewScript(`if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("pexpire", KEYS[1], ARGV[2]) else return 0 end`)
 
 func (db *DB) PExpireIE(key string, data interface{}, expiration time.Duration) error {
index 469e8c4..287115b 100644 (file)
@@ -24,14 +24,13 @@ package sdlgoredis_test
 
 import (
        "errors"
-       "strconv"
-       "testing"
-       "time"
-
        "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/sdlgoredis"
-       "github.com/go-redis/redis"
+       "github.com/go-redis/redis/v7"
        "github.com/stretchr/testify/assert"
        "github.com/stretchr/testify/mock"
+       "strconv"
+       "testing"
+       "time"
 )
 
 type clientMock struct {
@@ -138,6 +137,23 @@ func (m *clientMock) ScriptLoad(script string) *redis.StringCmd {
        return m.Called(script).Get(0).(*redis.StringCmd)
 }
 
+func (m *clientMock) Info(section ...string) *redis.StringCmd {
+       return m.Called(section).Get(0).(*redis.StringCmd)
+}
+
+type MockRedisSentinel struct {
+       mock.Mock
+}
+
+func (m *MockRedisSentinel) Master(name string) *redis.StringStringMapCmd {
+       a := m.Called(name)
+       return a.Get(0).(*redis.StringStringMapCmd)
+}
+func (m *MockRedisSentinel) Slaves(name string) *redis.SliceCmd {
+       a := m.Called(name)
+       return a.Get(0).(*redis.SliceCmd)
+}
+
 func setSubscribeNotifications() (*pubSubMock, sdlgoredis.SubscribeFn) {
        mock := new(pubSubMock)
        return mock, func(client sdlgoredis.RedisClient, channels ...string) sdlgoredis.Subscriber {
@@ -150,16 +166,48 @@ func (m *MockOS) Getenv(key string, defValue string) string {
        return a.String(0)
 }
 
-func setup(commandsExists bool) (*pubSubMock, *clientMock, *sdlgoredis.DB) {
-       mock := new(clientMock)
-       pubSubMock, subscribeNotifications := setSubscribeNotifications()
-       db := sdlgoredis.CreateDB(mock, subscribeNotifications)
+type setupEv struct {
+       pubSubMock []*pubSubMock
+       rClient    []*clientMock
+       rSentinel  []*MockRedisSentinel
+       db         []*sdlgoredis.DB
+}
+
+func setupHaEnv(commandsExists bool) (*pubSubMock, *clientMock, *sdlgoredis.DB) {
+       psm, cm, _, db := setupHaEnvWithSentinels(commandsExists)
+       return psm, cm, db
+}
+
+func setupHaEnvWithSentinels(commandsExists bool) (*pubSubMock, *clientMock, []*MockRedisSentinel, *sdlgoredis.DB) {
+       setupVals := setupEnv(
+               commandsExists,
+               "service-ricplt-dbaas-tcp-cluster-0.ricplt",
+               "6376",
+               "dbaasmaster",
+               "26376",
+               "",
+               "3",
+       )
+       return setupVals.pubSubMock[0], setupVals.rClient[0], setupVals.rSentinel, setupVals.db[0]
+}
+
+func setupSingleEnv(commandsExists bool) (*pubSubMock, *clientMock, *sdlgoredis.DB) {
+       setupVals := setupEnv(
+               commandsExists,
+               "service-ricplt-dbaas-tcp-cluster-0.ricplt",
+               "6376", "", "", "", "",
+       )
+       return setupVals.pubSubMock[0], setupVals.rClient[0], setupVals.db[0]
+}
+
+func setupEnv(commandsExists bool, host, port, msname, sntport, clsaddrlist, nodeCnt string) setupEv {
+       var ret setupEv
 
        dummyCommandInfo := redis.CommandInfo{
                Name: "dummy",
        }
-       cmdResult := make(map[string]*redis.CommandInfo, 0)
 
+       cmdResult := make(map[string]*redis.CommandInfo, 0)
        if commandsExists {
                cmdResult = map[string]*redis.CommandInfo{
                        "setie":    &dummyCommandInfo,
@@ -175,45 +223,41 @@ func setup(commandsExists bool) (*pubSubMock, *clientMock, *sdlgoredis.DB) {
                }
        }
 
-       mock.On("Command").Return(redis.NewCommandsInfoCmdResult(cmdResult, nil))
-       db.CheckCommands()
-       return pubSubMock, mock, db
-}
-
-func setupEnv(host, port, msname, sntport, clsaddrlist string) ([]*clientMock, []*sdlgoredis.DB) {
-       var clmocks []*clientMock
-
-       dummyCommandInfo := redis.CommandInfo{
-               Name: "dummy",
-       }
-       cmdResult := make(map[string]*redis.CommandInfo, 0)
-
-       cmdResult = map[string]*redis.CommandInfo{
-               "dummy": &dummyCommandInfo,
-       }
-
        osmock := new(MockOS)
        osmock.On("Getenv", "DBAAS_SERVICE_HOST", "localhost").Return(host)
        osmock.On("Getenv", "DBAAS_SERVICE_PORT", "6379").Return(port)
        osmock.On("Getenv", "DBAAS_MASTER_NAME", "").Return(msname)
        osmock.On("Getenv", "DBAAS_SERVICE_SENTINEL_PORT", "").Return(sntport)
        osmock.On("Getenv", "DBAAS_CLUSTER_ADDR_LIST", "").Return(clsaddrlist)
+       osmock.On("Getenv", "DBAAS_SERVICE_NODE_COUNT", "").Return(nodeCnt)
 
+       pubSubMock, subscribeNotifications := setSubscribeNotifications()
+       smock := new(MockRedisSentinel)
+       ret.rSentinel = append(ret.rSentinel, smock)
        clients := sdlgoredis.ReadConfigAndCreateDbClients(
                osmock,
                func(addr, port, clusterName string, isHa bool) sdlgoredis.RedisClient {
                        clm := new(clientMock)
                        clm.On("Command").Return(redis.NewCommandsInfoCmdResult(cmdResult, nil))
-                       clmocks = append(clmocks, clm)
+                       ret.rClient = append(ret.rClient, clm)
+                       ret.pubSubMock = append(ret.pubSubMock, pubSubMock)
                        return clm
                },
+               subscribeNotifications,
+               func(cfg *sdlgoredis.Config, addr string) *sdlgoredis.Sentinel {
+                       s := &sdlgoredis.Sentinel{
+                               IredisSentinelClient: smock,
+                               Cfg:                  cfg,
+                       }
+                       return s
+               },
        )
-
-       return clmocks, clients
+       ret.db = clients
+       return ret
 }
 
 func TestCloseDbSuccessfully(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        r.On("Close").Return(nil)
        err := db.CloseDB()
        assert.Nil(t, err)
@@ -221,7 +265,7 @@ func TestCloseDbSuccessfully(t *testing.T) {
 }
 
 func TestCloseDbFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        r.On("Close").Return(errors.New("Some error"))
        err := db.CloseDB()
        assert.NotNil(t, err)
@@ -229,7 +273,7 @@ func TestCloseDbFailure(t *testing.T) {
 }
 
 func TestMSetSuccessfully(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKeysAndValues := []interface{}{"key1", "value1", "key2", 2}
        r.On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil))
        err := db.MSet("key1", "value1", "key2", 2)
@@ -238,7 +282,7 @@ func TestMSetSuccessfully(t *testing.T) {
 }
 
 func TestMSetFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKeysAndValues := []interface{}{"key1", "value1", "key2", 2}
        r.On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", errors.New("Some error")))
        err := db.MSet("key1", "value1", "key2", 2)
@@ -247,7 +291,7 @@ func TestMSetFailure(t *testing.T) {
 }
 
 func TestMSetMPubSuccessfully(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"MSETMPUB", 2, 2, "key1", "val1", "key2", "val2",
                "chan1", "event1", "chan2", "event2"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult("", nil))
@@ -257,7 +301,7 @@ func TestMSetMPubSuccessfully(t *testing.T) {
 }
 
 func TestMsetMPubFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"MSETMPUB", 2, 2, "key1", "val1", "key2", "val2",
                "chan1", "event1", "chan2", "event2"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult("", errors.New("Some error")))
@@ -267,7 +311,7 @@ func TestMsetMPubFailure(t *testing.T) {
 }
 
 func TestMSetMPubCommandMissing(t *testing.T) {
-       _, r, db := setup(false)
+       _, r, db := setupHaEnv(false)
        expectedMessage := []interface{}{"MSETMPUB", 2, 2, "key1", "val1", "key2", "val2",
                "chan1", "event1", "chan2", "event2"}
        r.AssertNotCalled(t, "Do", expectedMessage)
@@ -278,7 +322,7 @@ func TestMSetMPubCommandMissing(t *testing.T) {
 }
 
 func TestMGetSuccessfully(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKeys := []string{"key1", "key2", "key3"}
        expectedResult := []interface{}{"val1", 2, nil}
        r.On("MGet", expectedKeys).Return(redis.NewSliceResult(expectedResult, nil))
@@ -289,7 +333,7 @@ func TestMGetSuccessfully(t *testing.T) {
 }
 
 func TestMGetFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKeys := []string{"key1", "key2", "key3"}
        expectedResult := []interface{}{nil}
        r.On("MGet", expectedKeys).Return(redis.NewSliceResult(expectedResult,
@@ -301,7 +345,7 @@ func TestMGetFailure(t *testing.T) {
 }
 
 func TestDelMPubSuccessfully(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"DELMPUB", 2, 2, "key1", "key2", "chan1", "event1",
                "chan2", "event2"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult("", nil))
@@ -311,7 +355,7 @@ func TestDelMPubSuccessfully(t *testing.T) {
 }
 
 func TestDelMPubFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"DELMPUB", 2, 2, "key1", "key2", "chan1", "event1",
                "chan2", "event2"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult("", errors.New("Some error")))
@@ -321,7 +365,7 @@ func TestDelMPubFailure(t *testing.T) {
 }
 
 func TestDelMPubCommandMissing(t *testing.T) {
-       _, r, db := setup(false)
+       _, r, db := setupHaEnv(false)
        expectedMessage := []interface{}{"DELMPUB", 2, 2, "key1", "key2", "chan1", "event1",
                "chan2", "event2"}
        r.AssertNotCalled(t, "Do", expectedMessage)
@@ -331,7 +375,7 @@ func TestDelMPubCommandMissing(t *testing.T) {
 }
 
 func TestDelSuccessfully(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKeys := []string{"key1", "key2"}
        r.On("Del", expectedKeys).Return(redis.NewIntResult(2, nil))
        assert.Nil(t, db.Del([]string{"key1", "key2"}))
@@ -339,7 +383,7 @@ func TestDelSuccessfully(t *testing.T) {
 }
 
 func TestDelFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKeys := []string{"key1", "key2"}
        r.On("Del", expectedKeys).Return(redis.NewIntResult(2, errors.New("Some error")))
        assert.NotNil(t, db.Del([]string{"key1", "key2"}))
@@ -347,7 +391,7 @@ func TestDelFailure(t *testing.T) {
 }
 
 func TestKeysSuccessfully(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedPattern := "pattern*"
        expectedResult := []string{"pattern1", "pattern2"}
        r.On("Keys", expectedPattern).Return(redis.NewStringSliceResult(expectedResult, nil))
@@ -358,7 +402,7 @@ func TestKeysSuccessfully(t *testing.T) {
 }
 
 func TestKeysFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedPattern := "pattern*"
        expectedResult := []string{}
        r.On("Keys", expectedPattern).Return(redis.NewStringSliceResult(expectedResult,
@@ -369,7 +413,7 @@ func TestKeysFailure(t *testing.T) {
 }
 
 func TestSetIEKeyExists(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"SETIE", "key", "newdata", "olddata"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult("OK", nil))
        result, err := db.SetIE("key", "olddata", "newdata")
@@ -379,7 +423,7 @@ func TestSetIEKeyExists(t *testing.T) {
 }
 
 func TestSetIEKeyDoesntExists(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"SETIE", "key", "newdata", "olddata"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult(nil, nil))
        result, err := db.SetIE("key", "olddata", "newdata")
@@ -389,7 +433,7 @@ func TestSetIEKeyDoesntExists(t *testing.T) {
 }
 
 func TestSetIEFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"SETIE", "key", "newdata", "olddata"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult(nil, errors.New("Some error")))
        result, err := db.SetIE("key", "olddata", "newdata")
@@ -399,7 +443,7 @@ func TestSetIEFailure(t *testing.T) {
 }
 
 func TestSetIECommandMissing(t *testing.T) {
-       _, r, db := setup(false)
+       _, r, db := setupHaEnv(false)
        expectedMessage := []interface{}{"SETIE", "key", "newdata", "olddata"}
        r.AssertNotCalled(t, "Do", expectedMessage)
        result, err := db.SetIE("key", "olddata", "newdata")
@@ -409,7 +453,7 @@ func TestSetIECommandMissing(t *testing.T) {
 }
 
 func TestSetIEPubKeyExists(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"SETIEMPUB", "key", "newdata", "olddata", "channel", "message"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult("OK", nil))
        result, err := db.SetIEPub([]string{"channel", "message"}, "key", "olddata", "newdata")
@@ -419,7 +463,7 @@ func TestSetIEPubKeyExists(t *testing.T) {
 }
 
 func TestSetIEPubKeyDoesntExists(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"SETIEMPUB", "key", "newdata", "olddata", "channel", "message"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult(nil, nil))
        result, err := db.SetIEPub([]string{"channel", "message"}, "key", "olddata", "newdata")
@@ -429,7 +473,7 @@ func TestSetIEPubKeyDoesntExists(t *testing.T) {
 }
 
 func TestSetIEPubFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"SETIEMPUB", "key", "newdata", "olddata", "channel", "message"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult(nil, errors.New("Some error")))
        result, err := db.SetIEPub([]string{"channel", "message"}, "key", "olddata", "newdata")
@@ -439,7 +483,7 @@ func TestSetIEPubFailure(t *testing.T) {
 }
 
 func TestSetIEPubCommandMissing(t *testing.T) {
-       _, r, db := setup(false)
+       _, r, db := setupHaEnv(false)
        expectedMessage := []interface{}{"SETIEMPUB", "key", "newdata", "olddata", "channel", "message"}
        r.AssertNotCalled(t, "Do", expectedMessage)
        result, err := db.SetIEPub([]string{"channel", "message"}, "key", "olddata", "newdata")
@@ -449,7 +493,7 @@ func TestSetIEPubCommandMissing(t *testing.T) {
 }
 
 func TestSetNXPubKeyDoesntExist(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"SETNXMPUB", "key", "data", "channel", "message"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult("OK", nil))
        result, err := db.SetNXPub([]string{"channel", "message"}, "key", "data")
@@ -459,7 +503,7 @@ func TestSetNXPubKeyDoesntExist(t *testing.T) {
 }
 
 func TestSetNXPubKeyExists(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"SETNXMPUB", "key", "data", "channel", "message"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult(nil, nil))
        result, err := db.SetNXPub([]string{"channel", "message"}, "key", "data")
@@ -469,7 +513,7 @@ func TestSetNXPubKeyExists(t *testing.T) {
 }
 
 func TestSetNXPubFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"SETNXMPUB", "key", "data", "channel", "message"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult(nil, errors.New("Some error")))
        result, err := db.SetNXPub([]string{"channel", "message"}, "key", "data")
@@ -479,7 +523,7 @@ func TestSetNXPubFailure(t *testing.T) {
 }
 
 func TestSetNXPubCommandMissing(t *testing.T) {
-       _, r, db := setup(false)
+       _, r, db := setupHaEnv(false)
        expectedMessage := []interface{}{"SETNXMPUB", "key", "data", "channel", "message"}
        r.AssertNotCalled(t, "Do", expectedMessage)
        result, err := db.SetNXPub([]string{"channel", "message"}, "key", "data")
@@ -489,7 +533,7 @@ func TestSetNXPubCommandMissing(t *testing.T) {
 }
 
 func TestSetNXSuccessfully(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedData := "data"
        r.On("SetNX", expectedKey, expectedData, time.Duration(0)).Return(redis.NewBoolResult(true, nil))
@@ -500,7 +544,7 @@ func TestSetNXSuccessfully(t *testing.T) {
 }
 
 func TestSetNXUnsuccessfully(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedData := "data"
        r.On("SetNX", expectedKey, expectedData, time.Duration(0)).Return(redis.NewBoolResult(false, nil))
@@ -511,7 +555,7 @@ func TestSetNXUnsuccessfully(t *testing.T) {
 }
 
 func TestSetNXFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedData := "data"
        r.On("SetNX", expectedKey, expectedData, time.Duration(0)).
@@ -523,7 +567,7 @@ func TestSetNXFailure(t *testing.T) {
 }
 
 func TestDelIEPubKeyDoesntExist(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"DELIEMPUB", "key", "data", "channel", "message"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult(int64(0), nil))
        result, err := db.DelIEPub([]string{"channel", "message"}, "key", "data")
@@ -533,7 +577,7 @@ func TestDelIEPubKeyDoesntExist(t *testing.T) {
 }
 
 func TestDelIEPubKeyExists(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"DELIEMPUB", "key", "data", "channel", "message"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult(int64(1), nil))
        result, err := db.DelIEPub([]string{"channel", "message"}, "key", "data")
@@ -543,7 +587,7 @@ func TestDelIEPubKeyExists(t *testing.T) {
 }
 
 func TestDelIEPubKeyExistsIntTypeRedisValue(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"DELIEMPUB", "key", "data", "channel", "message"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult(1, nil))
        result, err := db.DelIEPub([]string{"channel", "message"}, "key", "data")
@@ -553,7 +597,7 @@ func TestDelIEPubKeyExistsIntTypeRedisValue(t *testing.T) {
 }
 
 func TestDelIEPubFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"DELIEMPUB", "key", "data", "channel", "message"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult(int64(0), errors.New("Some error")))
        result, err := db.DelIEPub([]string{"channel", "message"}, "key", "data")
@@ -563,7 +607,7 @@ func TestDelIEPubFailure(t *testing.T) {
 }
 
 func TestDelIEPubCommandMissing(t *testing.T) {
-       _, r, db := setup(false)
+       _, r, db := setupHaEnv(false)
        expectedMessage := []interface{}{"DELIEMPUB", "key", "data", "channel", "message"}
        r.AssertNotCalled(t, "Do", expectedMessage)
        result, err := db.DelIEPub([]string{"channel", "message"}, "key", "data")
@@ -573,7 +617,7 @@ func TestDelIEPubCommandMissing(t *testing.T) {
 }
 
 func TestDelIEKeyDoesntExist(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"DELIE", "key", "data"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult(int64(0), nil))
        result, err := db.DelIE("key", "data")
@@ -583,7 +627,7 @@ func TestDelIEKeyDoesntExist(t *testing.T) {
 }
 
 func TestDelIEKeyExists(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"DELIE", "key", "data"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult(int64(1), nil))
        result, err := db.DelIE("key", "data")
@@ -593,7 +637,7 @@ func TestDelIEKeyExists(t *testing.T) {
 }
 
 func TestDelIEKeyExistsIntTypeRedisValue(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"DELIE", "key", "data"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult(1, nil))
        result, err := db.DelIE("key", "data")
@@ -603,7 +647,7 @@ func TestDelIEKeyExistsIntTypeRedisValue(t *testing.T) {
 }
 
 func TestDelIEFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedMessage := []interface{}{"DELIE", "key", "data"}
        r.On("Do", expectedMessage).Return(redis.NewCmdResult(int64(0), errors.New("Some error")))
        result, err := db.DelIE("key", "data")
@@ -613,7 +657,7 @@ func TestDelIEFailure(t *testing.T) {
 }
 
 func TestDelIECommandMissing(t *testing.T) {
-       _, r, db := setup(false)
+       _, r, db := setupHaEnv(false)
        expectedMessage := []interface{}{"DELIE", "key", "data"}
        r.AssertNotCalled(t, "Do", expectedMessage)
        result, err := db.DelIE("key", "data")
@@ -623,7 +667,7 @@ func TestDelIECommandMissing(t *testing.T) {
 }
 
 func TestSAddSuccessfully(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedData := []interface{}{"data", 2}
        r.On("SAdd", expectedKey, expectedData).Return(redis.NewIntResult(2, nil))
@@ -632,7 +676,7 @@ func TestSAddSuccessfully(t *testing.T) {
 }
 
 func TestSAddFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedData := []interface{}{"data", 2}
        r.On("SAdd", expectedKey, expectedData).Return(redis.NewIntResult(2, errors.New("Some error")))
@@ -641,7 +685,7 @@ func TestSAddFailure(t *testing.T) {
 }
 
 func TestSRemSuccessfully(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedData := []interface{}{"data", 2}
        r.On("SRem", expectedKey, expectedData).Return(redis.NewIntResult(2, nil))
@@ -650,7 +694,7 @@ func TestSRemSuccessfully(t *testing.T) {
 }
 
 func TestSRemFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedData := []interface{}{"data", 2}
        r.On("SRem", expectedKey, expectedData).Return(redis.NewIntResult(2, errors.New("Some error")))
@@ -659,7 +703,7 @@ func TestSRemFailure(t *testing.T) {
 }
 
 func TestSMembersSuccessfully(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedResult := []string{"member1", "member2"}
        r.On("SMembers", expectedKey).Return(redis.NewStringSliceResult(expectedResult, nil))
@@ -670,7 +714,7 @@ func TestSMembersSuccessfully(t *testing.T) {
 }
 
 func TestSMembersFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedResult := []string{"member1", "member2"}
        r.On("SMembers", expectedKey).Return(redis.NewStringSliceResult(expectedResult,
@@ -682,7 +726,7 @@ func TestSMembersFailure(t *testing.T) {
 }
 
 func TestSIsMemberIsMember(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedData := "data"
        r.On("SIsMember", expectedKey, expectedData).Return(redis.NewBoolResult(true, nil))
@@ -693,7 +737,7 @@ func TestSIsMemberIsMember(t *testing.T) {
 }
 
 func TestSIsMemberIsNotMember(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedData := "data"
        r.On("SIsMember", expectedKey, expectedData).Return(redis.NewBoolResult(false, nil))
@@ -704,7 +748,7 @@ func TestSIsMemberIsNotMember(t *testing.T) {
 }
 
 func TestSIsMemberFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedData := "data"
        r.On("SIsMember", expectedKey, expectedData).
@@ -716,7 +760,7 @@ func TestSIsMemberFailure(t *testing.T) {
 }
 
 func TestSCardSuccessfully(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        r.On("SCard", expectedKey).Return(redis.NewIntResult(1, nil))
        result, err := db.SCard("key")
@@ -726,7 +770,7 @@ func TestSCardSuccessfully(t *testing.T) {
 }
 
 func TestSCardFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        r.On("SCard", expectedKey).Return(redis.NewIntResult(1, errors.New("Some error")))
        result, err := db.SCard("key")
@@ -736,7 +780,7 @@ func TestSCardFailure(t *testing.T) {
 }
 
 func TestSubscribeChannelDBSubscribeRXUnsubscribe(t *testing.T) {
-       ps, r, db := setup(true)
+       ps, r, db := setupHaEnv(true)
        ch := make(chan *redis.Message)
        msg := redis.Message{
                Channel: "{prefix}channel",
@@ -762,7 +806,7 @@ func TestSubscribeChannelDBSubscribeRXUnsubscribe(t *testing.T) {
 }
 
 func TestSubscribeChannelDBSubscribeTwoUnsubscribeOne(t *testing.T) {
-       ps, r, db := setup(true)
+       ps, r, db := setupHaEnv(true)
        ch := make(chan *redis.Message)
        msg1 := redis.Message{
                Channel: "{prefix}channel1",
@@ -805,7 +849,7 @@ func TestSubscribeChannelDBSubscribeTwoUnsubscribeOne(t *testing.T) {
 }
 
 func TestSubscribeChannelReDBSubscribeAfterUnsubscribe(t *testing.T) {
-       ps, r, db := setup(true)
+       ps, r, db := setupHaEnv(true)
        ch := make(chan *redis.Message)
        msg := redis.Message{
                Channel: "{prefix}channel",
@@ -841,7 +885,7 @@ func TestSubscribeChannelReDBSubscribeAfterUnsubscribe(t *testing.T) {
 }
 
 func TestPTTLSuccessfully(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedResult := time.Duration(1)
        r.On("PTTL", expectedKey).Return(redis.NewDurationResult(expectedResult,
@@ -853,7 +897,7 @@ func TestPTTLSuccessfully(t *testing.T) {
 }
 
 func TestPTTLFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedResult := time.Duration(1)
        r.On("PTTL", expectedKey).Return(redis.NewDurationResult(expectedResult,
@@ -865,7 +909,7 @@ func TestPTTLFailure(t *testing.T) {
 }
 
 func TestPExpireIESuccessfully(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedData := "data"
        expectedDuration := strconv.FormatInt(int64(10000), 10)
@@ -879,7 +923,7 @@ func TestPExpireIESuccessfully(t *testing.T) {
 }
 
 func TestPExpireIEFailure(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedData := "data"
        expectedDuration := strconv.FormatInt(int64(10000), 10)
@@ -893,7 +937,7 @@ func TestPExpireIEFailure(t *testing.T) {
 }
 
 func TestPExpireIELockNotHeld(t *testing.T) {
-       _, r, db := setup(true)
+       _, r, db := setupHaEnv(true)
        expectedKey := "key"
        expectedData := "data"
        expectedDuration := strconv.FormatInt(int64(10000), 10)
@@ -907,73 +951,330 @@ func TestPExpireIELockNotHeld(t *testing.T) {
 }
 
 func TestClientStandaloneRedisLegacyEnv(t *testing.T) {
-       rcls, dbs := setupEnv(
-               "service-ricplt-dbaas-tcp-cluster-0.ricplt", "6376", "", "", "",
+       setupVals := setupEnv(
+               true,
+               "service-ricplt-dbaas-tcp-cluster-0.ricplt", "6376", "", "", "", "",
        )
-       assert.Equal(t, 1, len(rcls))
-       assert.Equal(t, 1, len(dbs))
+       assert.Equal(t, 1, len(setupVals.rClient))
+       assert.Equal(t, 1, len(setupVals.db))
 
        expectedKeysAndValues := []interface{}{"key1", "value1"}
-       rcls[0].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil))
-       err := dbs[0].MSet("key1", "value1")
+       setupVals.rClient[0].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil))
+       err := setupVals.db[0].MSet("key1", "value1")
        assert.Nil(t, err)
-       rcls[0].AssertExpectations(t)
+       setupVals.rClient[0].AssertExpectations(t)
 }
 
 func TestClientSentinelRedisLegacyEnv(t *testing.T) {
-       rcls, dbs := setupEnv(
-               "service-ricplt-dbaas-tcp-cluster-0.ricplt", "6376", "dbaasmaster", "26376", "",
+       setupVals := setupEnv(
+               true,
+               "service-ricplt-dbaas-tcp-cluster-0.ricplt", "6376", "dbaasmaster", "26376", "", "3",
        )
-       assert.Equal(t, 1, len(rcls))
-       assert.Equal(t, 1, len(dbs))
+       assert.Equal(t, 1, len(setupVals.rClient))
+       assert.Equal(t, 1, len(setupVals.db))
 
        expectedKeysAndValues := []interface{}{"key1", "value1"}
-       rcls[0].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil))
-       err := dbs[0].MSet("key1", "value1")
+       setupVals.rClient[0].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil))
+       err := setupVals.db[0].MSet("key1", "value1")
        assert.Nil(t, err)
-       rcls[0].AssertExpectations(t)
+       setupVals.rClient[0].AssertExpectations(t)
 }
 
 func TestClientTwoStandaloneRedisEnvs(t *testing.T) {
-       rcls, dbs := setupEnv(
+       setupVals := setupEnv(
+               true,
                "service-ricplt-dbaas-tcp-cluster-0.ricplt", "6376", "", "",
-               "service-ricplt-dbaas-tcp-cluster-0.ricplt,service-ricplt-dbaas-tcp-cluster-1.ricplt",
+               "service-ricplt-dbaas-tcp-cluster-0.ricplt,service-ricplt-dbaas-tcp-cluster-1.ricplt", "",
        )
-       assert.Equal(t, 2, len(rcls))
-       assert.Equal(t, 2, len(dbs))
+       assert.Equal(t, 2, len(setupVals.rClient))
+       assert.Equal(t, 2, len(setupVals.db))
 
        expectedKeysAndValues := []interface{}{"key1", "value1"}
-       rcls[0].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil))
-       err := dbs[0].MSet("key1", "value1")
+       setupVals.rClient[0].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil))
+       err := setupVals.db[0].MSet("key1", "value1")
        assert.Nil(t, err)
-       rcls[0].AssertExpectations(t)
+       setupVals.rClient[0].AssertExpectations(t)
 
        expectedKeysAndValues = []interface{}{"key2", "value2"}
-       rcls[1].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil))
-       err = dbs[1].MSet("key2", "value2")
+       setupVals.rClient[1].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil))
+       err = setupVals.db[1].MSet("key2", "value2")
        assert.Nil(t, err)
-       rcls[0].AssertExpectations(t)
-       rcls[1].AssertExpectations(t)
+       setupVals.rClient[0].AssertExpectations(t)
+       setupVals.rClient[1].AssertExpectations(t)
 }
 
 func TestClientTwoSentinelRedisEnvs(t *testing.T) {
-       rcls, dbs := setupEnv(
+       setupVals := setupEnv(
+               true,
                "service-ricplt-dbaas-tcp-cluster-0.ricplt", "6376", "dbaasmaster", "26376",
-               "service-ricplt-dbaas-tcp-cluster-0.ricplt,service-ricplt-dbaas-tcp-cluster-1.ricplt",
+               "service-ricplt-dbaas-tcp-cluster-0.ricplt,service-ricplt-dbaas-tcp-cluster-1.ricplt", "3",
        )
-       assert.Equal(t, 2, len(rcls))
-       assert.Equal(t, 2, len(dbs))
+       assert.Equal(t, 2, len(setupVals.rClient))
+       assert.Equal(t, 2, len(setupVals.db))
 
        expectedKeysAndValues := []interface{}{"key1", "value1"}
-       rcls[0].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil))
-       err := dbs[0].MSet("key1", "value1")
+       setupVals.rClient[0].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil))
+       err := setupVals.db[0].MSet("key1", "value1")
        assert.Nil(t, err)
-       rcls[0].AssertExpectations(t)
+       setupVals.rClient[0].AssertExpectations(t)
 
        expectedKeysAndValues = []interface{}{"key2", "value2"}
-       rcls[1].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil))
-       err = dbs[1].MSet("key2", "value2")
+       setupVals.rClient[1].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil))
+       err = setupVals.db[1].MSet("key2", "value2")
+       assert.Nil(t, err)
+       setupVals.rClient[0].AssertExpectations(t)
+       setupVals.rClient[1].AssertExpectations(t)
+}
+
+func TestInfoOfMasterRedisWithTwoSlavesSuccessfully(t *testing.T) {
+       _, r, db := setupHaEnv(true)
+       redisInfo := "# Replication\r\n" +
+               "role:master\r\n" +
+               "connected_slaves:2\r\n" +
+               "min_slaves_good_slaves:2\r\n" +
+               "slave0:ip=1.2.3.4,port=6379,state=online,offset=100200300,lag=0\r\n" +
+               "slave1:ip=5.6.7.8,port=6379,state=online,offset=100200300,lag=0\r\n"
+       expInfo := &sdlgoredis.DbInfo{
+               Fields: sdlgoredis.DbInfoFields{
+                       MasterRole:          true,
+                       ConnectedReplicaCnt: 2,
+               },
+       }
+
+       r.On("Info", []string{"all"}).Return(redis.NewStringResult(redisInfo, nil))
+       info, err := db.Info()
+       assert.Nil(t, err)
+       assert.Equal(t, expInfo, info)
+       r.AssertExpectations(t)
+}
+
+func TestInfoOfMasterRedisWithOneSlaveOnlineAndOtherSlaveNotOnlineSuccessfully(t *testing.T) {
+       _, r, db := setupHaEnv(true)
+       redisInfo := "# Replication\r\n" +
+               "role:master\r\n" +
+               "connected_slaves:1\r\n" +
+               "min_slaves_good_slaves:2\r\n" +
+               "slave0:ip=1.2.3.4,port=6379,state=online,offset=100200300,lag=0\r\n" +
+               "slave1:ip=5.6.7.8,port=6379,state=wait_bgsave,offset=100200300,lag=0\r\n"
+       expInfo := &sdlgoredis.DbInfo{
+               Fields: sdlgoredis.DbInfoFields{
+                       MasterRole:          true,
+                       ConnectedReplicaCnt: 1,
+               },
+       }
+
+       r.On("Info", []string{"all"}).Return(redis.NewStringResult(redisInfo, nil))
+       info, err := db.Info()
+       assert.Nil(t, err)
+       assert.Equal(t, expInfo, info)
+       r.AssertExpectations(t)
+}
+
+func TestInfoOfStandaloneMasterRedisSuccessfully(t *testing.T) {
+       _, r, db := setupHaEnv(true)
+       redisInfo := "# Replication\r\n" +
+               "role:master\r\n" +
+               "connected_slaves:0\r\n" +
+               "min_slaves_good_slaves:0\r\n"
+       expInfo := &sdlgoredis.DbInfo{
+               Fields: sdlgoredis.DbInfoFields{
+                       MasterRole:          true,
+                       ConnectedReplicaCnt: 0,
+               },
+       }
+
+       r.On("Info", []string{"all"}).Return(redis.NewStringResult(redisInfo, nil))
+       info, err := db.Info()
+       assert.Nil(t, err)
+       assert.Equal(t, expInfo, info)
+       r.AssertExpectations(t)
+}
+
+func TestInfoWithGibberishContentSuccessfully(t *testing.T) {
+       _, r, db := setupHaEnv(true)
+       redisInfo := "!#¤%&?+?´-\r\n"
+       expInfo := &sdlgoredis.DbInfo{}
+
+       r.On("Info", []string{"all"}).Return(redis.NewStringResult(redisInfo, nil))
+       info, err := db.Info()
+       assert.Nil(t, err)
+       assert.Equal(t, expInfo, info)
+       r.AssertExpectations(t)
+}
+
+func TestInfoWithEmptyContentSuccessfully(t *testing.T) {
+       _, r, db := setupHaEnv(true)
+       var redisInfo string
+       expInfo := &sdlgoredis.DbInfo{
+               Fields: sdlgoredis.DbInfoFields{
+                       MasterRole: false,
+               },
+       }
+
+       r.On("Info", []string{"all"}).Return(redis.NewStringResult(redisInfo, nil))
+       info, err := db.Info()
+       assert.Nil(t, err)
+       assert.Equal(t, expInfo, info)
+       r.AssertExpectations(t)
+}
+
+func TestStateWithMasterAndTwoSlaveRedisSuccessfully(t *testing.T) {
+       _, r, s, db := setupHaEnvWithSentinels(true)
+       redisMasterState := map[string]string{
+               "role-reported": "master",
+       }
+       redisSlavesState := make([]interface{}, 2)
+       redisSlavesState[0] = []interface{}{
+               "role-reported", "slave",
+               "ip", "10.20.30.40",
+               "port", "6379",
+               "flags", "slave",
+               "master-link-status", "up",
+       }
+       redisSlavesState[1] = []interface{}{
+               "master-link-status", "up",
+               "ip", "10.20.30.50",
+               "flags", "slave",
+               "port", "30000",
+               "role-reported", "slave",
+       }
+
+       expState := &sdlgoredis.DbState{
+               MasterDbState: sdlgoredis.MasterDbState{
+                       Fields: sdlgoredis.MasterDbStateFields{
+                               Role: "master",
+                       },
+               },
+               ReplicasDbState: &sdlgoredis.ReplicasDbState{
+                       States: []*sdlgoredis.ReplicaDbState{
+                               &sdlgoredis.ReplicaDbState{
+                                       Fields: sdlgoredis.ReplicaDbStateFields{
+                                               Role:             "slave",
+                                               Ip:               "10.20.30.40",
+                                               Port:             "6379",
+                                               MasterLinkStatus: "up",
+                                               Flags:            "slave",
+                                       },
+                               },
+                               &sdlgoredis.ReplicaDbState{
+                                       Fields: sdlgoredis.ReplicaDbStateFields{
+                                               Role:             "slave",
+                                               Ip:               "10.20.30.50",
+                                               Port:             "30000",
+                                               MasterLinkStatus: "up",
+                                               Flags:            "slave",
+                                       },
+                               },
+                       },
+               },
+       }
+
+       s[0].On("Master", "dbaasmaster").Return(redis.NewStringStringMapResult(redisMasterState, nil))
+       s[0].On("Slaves", "dbaasmaster").Return(redis.NewSliceResult(redisSlavesState, nil))
+       ret, err := db.State()
+       assert.Nil(t, err)
+       assert.Equal(t, expState, ret)
+       r.AssertExpectations(t)
+}
+
+func TestStateWithMasterAndOneSlaveRedisFailureInMasterRedisCall(t *testing.T) {
+       _, r, s, db := setupHaEnvWithSentinels(true)
+       redisMasterState := map[string]string{}
+       redisSlavesState := make([]interface{}, 1)
+       redisSlavesState[0] = []interface{}{
+               "role-reported", "slave",
+               "ip", "10.20.30.40",
+               "port", "6379",
+               "flags", "slave",
+               "master-link-status", "up",
+       }
+
+       expState := &sdlgoredis.DbState{
+               MasterDbState: sdlgoredis.MasterDbState{
+                       Err: errors.New("Some error"),
+               },
+               ReplicasDbState: &sdlgoredis.ReplicasDbState{
+                       States: []*sdlgoredis.ReplicaDbState{
+                               &sdlgoredis.ReplicaDbState{
+                                       Fields: sdlgoredis.ReplicaDbStateFields{
+                                               Role:             "slave",
+                                               Ip:               "10.20.30.40",
+                                               Port:             "6379",
+                                               MasterLinkStatus: "up",
+                                               Flags:            "slave",
+                                       },
+                               },
+                       },
+               },
+       }
+
+       s[0].On("Master", "dbaasmaster").Return(redis.NewStringStringMapResult(redisMasterState, errors.New("Some error")))
+       s[0].On("Slaves", "dbaasmaster").Return(redis.NewSliceResult(redisSlavesState, nil))
+       ret, err := db.State()
+       assert.NotNil(t, err)
+       assert.Equal(t, expState, ret)
+       r.AssertExpectations(t)
+}
+
+func TestStateWithMasterAndOneSlaveRedisFailureInSlavesRedisCall(t *testing.T) {
+       _, r, s, db := setupHaEnvWithSentinels(true)
+       redisMasterState := map[string]string{
+               "role-reported": "master",
+       }
+       redisSlavesState := make([]interface{}, 1)
+       redisSlavesState[0] = []interface{}{}
+
+       expState := &sdlgoredis.DbState{
+               MasterDbState: sdlgoredis.MasterDbState{
+                       Fields: sdlgoredis.MasterDbStateFields{
+                               Role: "master",
+                       },
+               },
+               ReplicasDbState: &sdlgoredis.ReplicasDbState{
+                       Err:    errors.New("Some error"),
+                       States: []*sdlgoredis.ReplicaDbState{},
+               },
+       }
+
+       s[0].On("Master", "dbaasmaster").Return(redis.NewStringStringMapResult(redisMasterState, nil))
+       s[0].On("Slaves", "dbaasmaster").Return(redis.NewSliceResult(redisSlavesState, errors.New("Some error")))
+       ret, err := db.State()
+       assert.NotNil(t, err)
+       assert.Equal(t, expState, ret)
+       r.AssertExpectations(t)
+}
+
+func TestStateWithSingleMasterRedisSuccessfully(t *testing.T) {
+       _, r, db := setupSingleEnv(true)
+       redisInfo := "# Replication\r\n" +
+               "role:master\r\n" +
+               "connected_slaves:0\r\n" +
+               "min_slaves_good_slaves:0\r\n"
+
+       expState := &sdlgoredis.DbState{
+               MasterDbState: sdlgoredis.MasterDbState{
+                       Fields: sdlgoredis.MasterDbStateFields{
+                               Role:  "master",
+                               Flags: "master",
+                       },
+               },
+       }
+
+       r.On("Info", []string{"all"}).Return(redis.NewStringResult(redisInfo, nil))
+       ret, err := db.State()
        assert.Nil(t, err)
-       rcls[0].AssertExpectations(t)
-       rcls[1].AssertExpectations(t)
+       assert.Equal(t, expState, ret)
+       r.AssertExpectations(t)
+}
+
+func TestStateWithSingleMasterRedisFailureInInfoCall(t *testing.T) {
+       _, r, db := setupSingleEnv(true)
+       redisInfo := ""
+       expState := &sdlgoredis.DbState{}
+
+       r.On("Info", []string{"all"}).Return(redis.NewStringResult(redisInfo, errors.New("Some error")))
+       ret, err := db.State()
+       assert.NotNil(t, err)
+       assert.Equal(t, expState, ret)
+       r.AssertExpectations(t)
 }
diff --git a/internal/sdlgoredis/sdlgosentinel.go b/internal/sdlgoredis/sdlgosentinel.go
new file mode 100644 (file)
index 0000000..7af0410
--- /dev/null
@@ -0,0 +1,111 @@
+/*
+   Copyright (c) 2021 AT&T Intellectual Property.
+   Copyright (c) 2018-2021 Nokia.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+ * This source code is part of the near-RT RIC (RAN Intelligent Controller)
+ * platform project (RICP).
+ */
+
+package sdlgoredis
+
+import (
+       "github.com/go-redis/redis/v7"
+)
+
+type Sentinel struct {
+       IredisSentinelClient
+       Cfg *Config
+}
+
+type IredisSentinelClient interface {
+       Master(name string) *redis.StringStringMapCmd
+       Slaves(name string) *redis.SliceCmd
+}
+
+type RedisSentinelCreateCb func(cfg *Config, addr string) *Sentinel
+
+func newRedisSentinel(cfg *Config, addr string) *Sentinel {
+       redisAddress := addr + ":" + cfg.sentinelPort
+       return &Sentinel{
+               IredisSentinelClient: redis.NewSentinelClient(&redis.Options{
+                       Addr:       redisAddress,
+                       Password:   "", // no password set
+                       DB:         0,  // use default DB
+                       PoolSize:   20,
+                       MaxRetries: 2,
+               }),
+               Cfg: cfg,
+       }
+}
+
+func (s *Sentinel) GetDbState() (*DbState, error) {
+       state := new(DbState)
+       mState, mErr := s.getMasterDbState()
+       rState, rErr := s.getReplicasState()
+       state.MasterDbState = *mState
+       state.ReplicasDbState = rState
+       if mErr == nil {
+               return state, rErr
+       }
+       return state, mErr
+}
+
+func (s *Sentinel) getMasterDbState() (*MasterDbState, error) {
+       state := new(MasterDbState)
+       redisVal, redisErr := s.Master(s.Cfg.masterName).Result()
+       if redisErr == nil {
+               state.Fields.Ip = redisVal["ip"]
+               state.Fields.Port = redisVal["port"]
+               state.Fields.Flags = redisVal["flags"]
+               state.Fields.Role = redisVal["role-reported"]
+       }
+       state.Err = redisErr
+       return state, redisErr
+}
+
+func (s *Sentinel) getReplicasState() (*ReplicasDbState, error) {
+       states := new(ReplicasDbState)
+       states.States = make([]*ReplicaDbState, 0)
+
+       redisVal, redisErr := s.Slaves(s.Cfg.masterName).Result()
+       if redisErr == nil {
+               for _, redisSlave := range redisVal {
+                       replicaState := readReplicaState(redisSlave.([]interface{}))
+                       states.States = append(states.States, replicaState)
+               }
+       }
+       states.Err = redisErr
+       return states, redisErr
+}
+
+func readReplicaState(redisSlaves []interface{}) *ReplicaDbState {
+       state := new(ReplicaDbState)
+       for i := 0; i < len(redisSlaves); i += 2 {
+               if redisSlaves[i].(string) == "ip" {
+                       state.Fields.Ip = redisSlaves[i+1].(string)
+               } else if redisSlaves[i].(string) == "port" {
+                       state.Fields.Port = redisSlaves[i+1].(string)
+               } else if redisSlaves[i].(string) == "flags" {
+                       state.Fields.Flags = redisSlaves[i+1].(string)
+               } else if redisSlaves[i].(string) == "role-reported" {
+                       state.Fields.Role = redisSlaves[i+1].(string)
+               } else if redisSlaves[i].(string) == "master-link-status" {
+                       state.Fields.MasterLinkStatus = redisSlaves[i+1].(string)
+               }
+       }
+       return state
+}