From 977a55ca96d5dba1c7f9273671747eaf9cd6f894 Mon Sep 17 00:00:00 2001 From: Timo Tietavainen Date: Tue, 26 Oct 2021 09:11:53 +0300 Subject: [PATCH] Implement SDL CLI 'healthcheck' -command Implement a new 'healthcheck' -command for SDL CLI to validate healthiness of the SDL database. Under the hood 'healthcheck' -command calls Redis client's 'Master' and 'Slaves' APIs and parses from the responses master and slave Redis servers related fields to show the overall healthiness of the SDL database backend. Redis client's version was upgraded from v6.15.9 to v7 v7.4.1, because old client didn't support 'Master' and 'Slaves' Redis sentinel APIs. Also implement usage of Redis client's 'NewSentinelClient' client connection to read state information via 'Master' and 'Slaves' APIs. In case of standalone Redis server deployment state information is read via Redis client's 'Info' API. Issue-Id: RIC-113 Signed-off-by: Timo Tietavainen Change-Id: I7d964ac5c3d10cc90075977e2b11da7a9c11949d --- go.mod | 2 +- go.sum | 11 +- internal/cli/cli_private_fn_test.go | 32 ++ internal/cli/healthcheck.go | 106 ++++++ internal/cli/healthcheck_test.go | 188 ++++++++++ internal/cli/root.go | 1 + internal/cli/types.go | 19 +- internal/cli/utils.go | 35 ++ internal/mocks/db_mocks_private_testing.go | 42 +++ internal/sdlgoredis/dbinfo.go | 36 ++ internal/sdlgoredis/dbstate.go | 138 +++++++ internal/sdlgoredis/dbstate_test.go | 187 ++++++++++ internal/sdlgoredis/sdlgoredis.go | 119 ++++++- internal/sdlgoredis/sdlgoredis_test.go | 553 ++++++++++++++++++++++------- internal/sdlgoredis/sdlgosentinel.go | 111 ++++++ 15 files changed, 1434 insertions(+), 146 deletions(-) create mode 100644 internal/cli/cli_private_fn_test.go create mode 100644 internal/cli/healthcheck.go create mode 100644 internal/cli/healthcheck_test.go create mode 100644 internal/cli/utils.go create mode 100644 internal/mocks/db_mocks_private_testing.go create mode 100644 internal/sdlgoredis/dbinfo.go create mode 100644 internal/sdlgoredis/dbstate.go create mode 100644 internal/sdlgoredis/dbstate_test.go create mode 100644 internal/sdlgoredis/sdlgosentinel.go diff --git a/go.mod b/go.mod index 7f50555..8085e16 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module gerrit.o-ran-sc.org/r/ric-plt/sdlgo go 1.12 require ( - github.com/go-redis/redis v6.15.9+incompatible + github.com/go-redis/redis/v7 v7.4.1 github.com/onsi/ginkgo v1.14.0 // indirect github.com/spf13/cobra v1.1.1 github.com/stretchr/testify v1.3.0 diff --git a/go.sum b/go.sum index d5ee04e..fcfcb96 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= -github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOrTI= +github.com/go-redis/redis/v7 v7.4.1/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -140,9 +140,11 @@ github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= @@ -235,6 +237,7 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -261,6 +264,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= @@ -321,8 +325,9 @@ google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyz google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/internal/cli/cli_private_fn_test.go b/internal/cli/cli_private_fn_test.go new file mode 100644 index 0000000..831b5f9 --- /dev/null +++ b/internal/cli/cli_private_fn_test.go @@ -0,0 +1,32 @@ +/* + Copyright (c) 2021 AT&T Intellectual Property. + Copyright (c) 2018-2021 Nokia. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + * This source code is part of the near-RT RIC (RAN Intelligent Controller) + * platform project (RICP). + */ + +package cli + +import ( + "github.com/spf13/cobra" +) + +// NewHealthCheckCmdForTest is used only in unit tests to mock database. +func NewHealthCheckCmdForTest(dbCreateCb DbCreateCb) *cobra.Command { + return newHealthCheckCmd(dbCreateCb) +} diff --git a/internal/cli/healthcheck.go b/internal/cli/healthcheck.go new file mode 100644 index 0000000..94467a9 --- /dev/null +++ b/internal/cli/healthcheck.go @@ -0,0 +1,106 @@ +/* + Copyright (c) 2021 AT&T Intellectual Property. + Copyright (c) 2018-2021 Nokia. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + * This source code is part of the near-RT RIC (RAN Intelligent Controller) + * platform project (RICP). + */ + +package cli + +import ( + "bytes" + "fmt" + "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/sdlgoredis" + "github.com/spf13/cobra" + "os" +) + +func NewHealthCheckCmd() *cobra.Command { + return newHealthCheckCmd(newDatabase) +} + +func newHealthCheckCmd(dbCreateCb DbCreateCb) *cobra.Command { + cmd := &cobra.Command{ + Use: "healthcheck", + Short: "healthcheck - validates database healthiness", + Long: `healthcheck - validates database healthiness`, + RunE: func(cmd *cobra.Command, args []string) error { + var buf bytes.Buffer + sdlgoredis.SetDbLogger(&buf) + out, err := runHealthCheck(dbCreateCb) + cmd.Println(out) + if err != nil { + cmd.PrintErrf("%s\n", buf.String()) + } + return err + }, + } + cmd.SetOut(os.Stdout) + return cmd +} + +func runHealthCheck(dbCreateCb DbCreateCb) (string, error) { + var anyErr error + var str string + var states []sdlgoredis.DbState + for _, dbInst := range dbCreateCb().Instances { + info, err := dbInst.State() + if err != nil { + anyErr = fmt.Errorf("SDL CLI error: %v", err) + } + states = append(states, *info) + } + str = writeStateResults(states) + return str, anyErr +} + +func writeStateResults(dbStates []sdlgoredis.DbState) string { + var str string + var anyErr error + for i, dbState := range dbStates { + if err := dbState.IsOnline(); err != nil { + anyErr = err + } + str = str + fmt.Sprintf(" SDL DB backend #%d\n", (i+1)) + mAddr := dbState.MasterDbState.GetAddress() + err := dbState.MasterDbState.IsOnline() + if err == nil { + str = str + fmt.Sprintf(" Master (%s): OK\n", mAddr) + } else { + str = str + fmt.Sprintf(" Master (%s): NOK\n", mAddr) + str = str + fmt.Sprintf(" %s\n", err.Error()) + } + if dbState.ReplicasDbState != nil { + for j, rInfo := range dbState.ReplicasDbState.States { + err := rInfo.IsOnline() + if err == nil { + str = str + fmt.Sprintf(" Replica #%d (%s): OK\n", (j+1), rInfo.GetAddress()) + } else { + str = str + fmt.Sprintf(" Replica #%d (%s): NOK\n", (j+1), rInfo.GetAddress()) + str = str + fmt.Sprintf(" %s\n", err.Error()) + } + } + } + } + if anyErr == nil { + str = fmt.Sprintf("Overall status: OK\n\n") + str + } else { + str = fmt.Sprintf("Overall status: NOK\n\n") + str + } + return str +} diff --git a/internal/cli/healthcheck_test.go b/internal/cli/healthcheck_test.go new file mode 100644 index 0000000..be98aed --- /dev/null +++ b/internal/cli/healthcheck_test.go @@ -0,0 +1,188 @@ +/* + Copyright (c) 2021 AT&T Intellectual Property. + Copyright (c) 2018-2021 Nokia. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + * This source code is part of the near-RT RIC (RAN Intelligent Controller) + * platform project (RICP). + */ + +package cli_test + +import ( + "bytes" + "errors" + "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/cli" + "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/mocks" + "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/sdlgoredis" + "github.com/stretchr/testify/assert" + "testing" +) + +var hcMocks *healthCheckMocks + +type healthCheckMocks struct { + dbIface *mocks.MockDB + dbErr error + dbState sdlgoredis.DbState +} + +func setupHcMockMasterDb(ip, port string, replicas uint32) { + hcMocks = new(healthCheckMocks) + hcMocks.dbState.MasterDbState.Fields.Role = "master" + hcMocks.dbState.MasterDbState.Fields.Ip = ip + hcMocks.dbState.MasterDbState.Fields.Port = port + hcMocks.dbState.MasterDbState.Fields.Flags = "master" +} + +func setupHcMockReplicaDb() { + hcMocks = new(healthCheckMocks) + hcMocks.dbState.ReplicasDbState = new(sdlgoredis.ReplicasDbState) + hcMocks.dbState.ReplicasDbState.States = []*sdlgoredis.ReplicaDbState{ + &sdlgoredis.ReplicaDbState{ + Fields: sdlgoredis.ReplicaDbStateFields{ + Role: "slave", + }, + }, + } +} + +func addHcMockReplicaDbState(ip, port, masterLinkOk string) { + if hcMocks.dbState.ReplicasDbState == nil { + hcMocks.dbState.ReplicasDbState = new(sdlgoredis.ReplicasDbState) + } + hcMocks.dbState.ReplicasDbState.States = append(hcMocks.dbState.ReplicasDbState.States, + &sdlgoredis.ReplicaDbState{ + Fields: sdlgoredis.ReplicaDbStateFields{ + Role: "slave", + Ip: ip, + Port: port, + MasterLinkStatus: masterLinkOk, + Flags: "slave", + }, + }, + ) +} + +func newMockDatabase() *cli.Database { + db := &cli.Database{} + hcMocks.dbIface = new(mocks.MockDB) + hcMocks.dbIface.On("State").Return(&hcMocks.dbState, hcMocks.dbErr) + db.Instances = append(db.Instances, hcMocks.dbIface) + return db +} + +func runHcCli() (string, error) { + buf := new(bytes.Buffer) + cmd := cli.NewHealthCheckCmdForTest(newMockDatabase) + cmd.SetOut(buf) + + err := cmd.Execute() + + return buf.String(), err +} + +func TestCliHealthCheckCanShowHelp(t *testing.T) { + var expOkErr error + expHelp := "Usage:\n " + "healthcheck [flags]" + expNokErr := errors.New("unknown flag: --some-unknown-flag") + tests := []struct { + args string + expErr error + expOutput string + }{ + {args: "-h", expErr: expOkErr, expOutput: expHelp}, + {args: "--help", expErr: expOkErr, expOutput: expHelp}, + {args: "--some-unknown-flag", expErr: expNokErr, expOutput: expHelp}, + } + + for _, test := range tests { + buf := new(bytes.Buffer) + cmd := cli.NewHealthCheckCmd() + cmd.SetOut(buf) + cmd.SetArgs([]string{test.args}) + + err := cmd.Execute() + + stdout := buf.String() + assert.Equal(t, test.expErr, err) + assert.Contains(t, stdout, test.expOutput) + } +} + +func TestCliHealthCheckCanShowHaDeploymentOkStatusCorrectly(t *testing.T) { + setupHcMockMasterDb("10.20.30.40", "6379", 2) + addHcMockReplicaDbState("1.2.3.4", "6379", "ok") + addHcMockReplicaDbState("5.6.7.8", "6379", "ok") + + stdout, err := runHcCli() + + assert.Nil(t, err) + assert.Contains(t, stdout, "Overall status: OK") + assert.Contains(t, stdout, "Master (10.20.30.40:6379): OK") + assert.Contains(t, stdout, "Replica #1 (1.2.3.4:6379): OK") + assert.Contains(t, stdout, "Replica #2 (5.6.7.8:6379): OK") + +} + +func TestCliHealthCheckCanShowHaDeploymentStatusCorrectlyWhenOneReplicaStateNotUp(t *testing.T) { + setupHcMockMasterDb("10.20.30.40", "6379", 2) + addHcMockReplicaDbState("1.2.3.4", "6379", "ok") + addHcMockReplicaDbState("5.6.7.8", "6379", "nok") + + stdout, err := runHcCli() + + assert.Nil(t, err) + assert.Contains(t, stdout, "Overall status: NOK") + assert.Contains(t, stdout, "Replica #2 (5.6.7.8:6379): NOK") + assert.Contains(t, stdout, "Replica link to the master is down") +} + +func TestCliHealthCheckCanShowHaDeploymentStatusCorrectlyWhenDbStateQueryFails(t *testing.T) { + setupHcMockMasterDb("10.20.30.40", "6379", 0) + hcMocks.dbErr = errors.New("Some error") + expCliErr := errors.New("SDL CLI error: Some error") + + buf := new(bytes.Buffer) + cmd := cli.NewHealthCheckCmdForTest(newMockDatabase) + cmd.SetErr(buf) + + err := cmd.Execute() + stderr := buf.String() + + assert.Equal(t, expCliErr, err) + assert.Contains(t, stderr, "Error: "+expCliErr.Error()) +} + +func TestCliHealthCheckCanShowHaDeploymentOkStatusCorrectlyWhenDbStateIsFromReplicaServer(t *testing.T) { + setupHcMockReplicaDb() + + stdout, err := runHcCli() + + assert.Nil(t, err) + assert.Contains(t, stdout, "Overall status: NOK") + assert.Contains(t, stdout, "Master (): NOK") +} + +func TestCliHealthCheckCanShowStandaloneDeploymentOkStatusCorrectly(t *testing.T) { + setupHcMockMasterDb("10.20.30.40", "6379", 0) + + stdout, err := runHcCli() + + assert.Nil(t, err) + assert.Contains(t, stdout, "Overall status: OK") + assert.Contains(t, stdout, "Master (10.20.30.40:6379): OK") +} diff --git a/internal/cli/root.go b/internal/cli/root.go index ceef6c2..6d20ebc 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -34,5 +34,6 @@ func NewRootCmd() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { }, } + cmd.AddCommand(NewHealthCheckCmd()) return cmd } diff --git a/internal/cli/types.go b/internal/cli/types.go index ad942c8..de99e62 100644 --- a/internal/cli/types.go +++ b/internal/cli/types.go @@ -22,5 +22,22 @@ package cli -//Name of the SDL CLI application +import "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/sdlgoredis" + +//iDatabase is an interface towards database backend, for the time being +//sdlgoredis.DB implements this interface. +type iDatabase interface { + Info() (*sdlgoredis.DbInfo, error) + State() (*sdlgoredis.DbState, error) +} + +//Database struct is a holder for the internal database instances. +type Database struct { + Instances []iDatabase +} + +//DbCreateCb callback function type to create a new database +type DbCreateCb func() *Database + +//SdlCliApp constant defines the name of the SDL CLI application const SdlCliApp = "sdlcli" diff --git a/internal/cli/utils.go b/internal/cli/utils.go new file mode 100644 index 0000000..3702050 --- /dev/null +++ b/internal/cli/utils.go @@ -0,0 +1,35 @@ +/* + Copyright (c) 2021 AT&T Intellectual Property. + Copyright (c) 2018-2021 Nokia. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + * This source code is part of the near-RT RIC (RAN Intelligent Controller) + * platform project (RICP). + */ + +package cli + +import ( + "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/sdlgoredis" +) + +func newDatabase() *Database { + db := &Database{} + for _, v := range sdlgoredis.Create() { + db.Instances = append(db.Instances, v) + } + return db +} diff --git a/internal/mocks/db_mocks_private_testing.go b/internal/mocks/db_mocks_private_testing.go new file mode 100644 index 0000000..7832ee3 --- /dev/null +++ b/internal/mocks/db_mocks_private_testing.go @@ -0,0 +1,42 @@ +/* + Copyright (c) 2021 AT&T Intellectual Property. + Copyright (c) 2018-2021 Nokia. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + * This source code is part of the near-RT RIC (RAN Intelligent Controller) + * platform project (RICP). + */ + +package mocks + +import ( + "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/sdlgoredis" + "github.com/stretchr/testify/mock" +) + +type MockDB struct { + mock.Mock +} + +func (m *MockDB) Info() (*sdlgoredis.DbInfo, error) { + a := m.Called() + return a.Get(0).(*sdlgoredis.DbInfo), a.Error(1) +} + +func (m *MockDB) State() (*sdlgoredis.DbState, error) { + a := m.Called() + return a.Get(0).(*sdlgoredis.DbState), a.Error(1) +} diff --git a/internal/sdlgoredis/dbinfo.go b/internal/sdlgoredis/dbinfo.go new file mode 100644 index 0000000..b8990e0 --- /dev/null +++ b/internal/sdlgoredis/dbinfo.go @@ -0,0 +1,36 @@ +/* + Copyright (c) 2021 AT&T Intellectual Property. + Copyright (c) 2018-2021 Nokia. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + * This source code is part of the near-RT RIC (RAN Intelligent Controller) + * platform project (RICP). + */ + +package sdlgoredis + +//DbInfo struct is a holder for DB information, which is received from +//sdlgoredis 'info' call's output. +type DbInfo struct { + Fields DbInfoFields +} + +//DbInfoFields struct is a holder for fields, which are read from sdlgoredis +//'info' call's output. +type DbInfoFields struct { + MasterRole bool + ConnectedReplicaCnt uint32 +} diff --git a/internal/sdlgoredis/dbstate.go b/internal/sdlgoredis/dbstate.go new file mode 100644 index 0000000..543f22e --- /dev/null +++ b/internal/sdlgoredis/dbstate.go @@ -0,0 +1,138 @@ +/* + Copyright (c) 2021 AT&T Intellectual Property. + Copyright (c) 2018-2021 Nokia. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + * This source code is part of the near-RT RIC (RAN Intelligent Controller) + * platform project (RICP). + */ + +package sdlgoredis + +import ( + "fmt" +) + +//DbState struct is a holder for DB state information, which is received from +//sdlgoredis sentinel 'Master' and 'Slaves' calls output. +type DbState struct { + MasterDbState MasterDbState + ReplicasDbState *ReplicasDbState +} + +//MasterDbState struct is a holder for master Redis state information. +type MasterDbState struct { + Err error + Fields MasterDbStateFields +} + +//ReplicasDbState struct is a holder for Redis slaves state information. +type ReplicasDbState struct { + Err error + States []*ReplicaDbState +} + +//ReplicaDbState struct is a holder for one Redis slave state information. +type ReplicaDbState struct { + Fields ReplicaDbStateFields +} + +//MasterDbStateFields struct is a holder for master Redis state information +//fields which are read from sdlgoredis sentinel 'Master' call output. +type MasterDbStateFields struct { + Role string + Ip string + Port string + Flags string +} + +//ReplicaDbStateFields struct is a holder for slave Redis state information +//fields which are read from sdlgoredis sentinel 'Slaves' call output. +type ReplicaDbStateFields struct { + Role string + Ip string + Port string + MasterLinkStatus string + Flags string +} + +func (dbst *DbState) IsOnline() error { + if err := dbst.MasterDbState.IsOnline(); err != nil { + return err + } + if dbst.ReplicasDbState != nil { + if err := dbst.ReplicasDbState.IsOnline(); err != nil { + return err + } + } + return nil +} + +func (mdbst *MasterDbState) IsOnline() error { + if mdbst.Err != nil { + return mdbst.Err + } + if mdbst.Fields.Role != "master" { + return fmt.Errorf("No master DB, current role '%s'", mdbst.Fields.Role) + } + if mdbst.Fields.Flags != "master" { + return fmt.Errorf("Master flags are '%s', expected 'master'", mdbst.Fields.Flags) + } + return nil +} + +func (mdbst *MasterDbState) GetAddress() string { + if mdbst.Fields.Ip != "" || mdbst.Fields.Port != "" { + return mdbst.Fields.Ip + ":" + mdbst.Fields.Port + } else { + return "" + } +} + +func (rdbst *ReplicasDbState) IsOnline() error { + if rdbst.Err != nil { + return rdbst.Err + } + for _, state := range rdbst.States { + if err := state.IsOnline(); err != nil { + return err + } + } + return nil +} + +func (rdbst *ReplicaDbState) IsOnline() error { + if rdbst.Fields.Role != "slave" { + return fmt.Errorf("Replica role is '%s', expected 'slave'", rdbst.Fields.Role) + } + + if rdbst.Fields.MasterLinkStatus != "ok" { + return fmt.Errorf("Replica link to the master is down") + } + + if rdbst.Fields.Flags != "slave" { + return fmt.Errorf("Replica flags are '%s', expected 'slave'", rdbst.Fields.Flags) + } + return nil +} + +func (rdbst *ReplicaDbState) GetAddress() string { + if rdbst.Fields.Ip != "" || rdbst.Fields.Port != "" { + return rdbst.Fields.Ip + ":" + rdbst.Fields.Port + } else { + return "" + } +} diff --git a/internal/sdlgoredis/dbstate_test.go b/internal/sdlgoredis/dbstate_test.go new file mode 100644 index 0000000..0b03afa --- /dev/null +++ b/internal/sdlgoredis/dbstate_test.go @@ -0,0 +1,187 @@ +/* + Copyright (c) 2021 AT&T Intellectual Property. + Copyright (c) 2018-2021 Nokia. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + * This source code is part of the near-RT RIC (RAN Intelligent Controller) + * platform project (RICP). + */ + +package sdlgoredis_test + +import ( + "errors" + "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/sdlgoredis" + "github.com/stretchr/testify/assert" + "testing" +) + +type dbStateMock struct { + state sdlgoredis.DbState +} + +func setupDbState() *dbStateMock { + return new(dbStateMock) +} + +func (ds *dbStateMock) setMasterError(err error) { + ds.state.MasterDbState.Err = err +} + +func (ds *dbStateMock) setMasterFields(role, ip, port, rCnt, flags string) { + ds.state.MasterDbState.Fields.Role = role + ds.state.MasterDbState.Fields.Ip = ip + ds.state.MasterDbState.Fields.Port = port + ds.state.MasterDbState.Fields.Flags = flags +} + +func (ds *dbStateMock) setReplicaError(err error) { + if ds.state.ReplicasDbState == nil { + ds.state.ReplicasDbState = new(sdlgoredis.ReplicasDbState) + } + ds.state.ReplicasDbState.Err = err +} + +func (ds *dbStateMock) addReplicaFields(role, ip, port, mls, flags string) { + if ds.state.ReplicasDbState == nil { + ds.state.ReplicasDbState = new(sdlgoredis.ReplicasDbState) + } + newState := new(sdlgoredis.ReplicaDbState) + newState.Fields.Role = role + newState.Fields.Ip = ip + newState.Fields.Port = port + newState.Fields.MasterLinkStatus = mls + newState.Fields.Flags = flags + ds.state.ReplicasDbState.States = append(ds.state.ReplicasDbState.States, newState) +} + +func TestIsOnlineWhenSingleMasterSuccessfully(t *testing.T) { + st := setupDbState() + st.setMasterFields("master", "1.2.3.4", "60000", "0", "master") + err := st.state.IsOnline() + assert.Nil(t, err) +} + +func TestIsOnlineWhenSingleMasterFailureIfErrorHasSet(t *testing.T) { + testErr := errors.New("Some error") + st := setupDbState() + st.setMasterFields("master", "1.2.3.4", "60000", "0", "master") + st.setMasterError(testErr) + err := st.state.IsOnline() + assert.Equal(t, testErr, err) +} + +func TestIsOnlineWhenSingleMasterFailureIfNotMasterRole(t *testing.T) { + expErr := errors.New("No master DB, current role 'not-master'") + st := setupDbState() + st.setMasterFields("not-master", "1.2.3.4", "60000", "0", "master") + err := st.state.IsOnline() + assert.Equal(t, expErr, err) +} + +func TestIsOnlineWhenSingleMasterFailureIfErrorFlags(t *testing.T) { + expErr := errors.New("Master flags are 'any-error,master', expected 'master'") + st := setupDbState() + st.setMasterFields("master", "1.2.3.4", "60000", "0", "any-error,master") + err := st.state.IsOnline() + assert.Equal(t, expErr, err) +} + +func TestGetAddressMasterSuccessfully(t *testing.T) { + st := setupDbState() + st.setMasterFields("master", "1.2.3.4", "60000", "0", "master") + addr := st.state.MasterDbState.GetAddress() + assert.Equal(t, "1.2.3.4:60000", addr) +} + +func TestGetAddressMasterFailureNoIpPort(t *testing.T) { + st := setupDbState() + st.setMasterFields("master", "", "", "0", "master") + addr := st.state.MasterDbState.GetAddress() + assert.Equal(t, "", addr) +} + +func TestIsOnlineWhenMasterAndTwoReplicasSuccessfully(t *testing.T) { + st := setupDbState() + st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") + st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave") + st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave") + err := st.state.IsOnline() + assert.Nil(t, err) +} + +func TestIsOnlineWhenMasterAndTwoReplicasFailureIfErrorHasSet(t *testing.T) { + testErr := errors.New("Some error") + st := setupDbState() + st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") + st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave") + st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave") + st.setReplicaError(testErr) + err := st.state.IsOnline() + assert.Equal(t, testErr, err) +} + +func TestIsOnlineWhenMasterAndTwoReplicasFailureIfNotSlaveRole(t *testing.T) { + expErr := errors.New("Replica role is 'not-slave', expected 'slave'") + st := setupDbState() + st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") + st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave") + st.addReplicaFields("not-slave", "6.7.8.10", "3450", "ok", "slave") + err := st.state.IsOnline() + assert.Equal(t, expErr, err) +} + +func TestIsOnlineWhenMasterAndTwoReplicasFailureIfMasterLinkDown(t *testing.T) { + expErr := errors.New("Replica link to the master is down") + st := setupDbState() + st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") + st.addReplicaFields("slave", "6.7.8.9", "1234", "nok", "slave") + st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave") + err := st.state.IsOnline() + assert.Equal(t, expErr, err) +} + +func TestIsOnlineWhenMasterAndTwoReplicasFailureIfErrorFlags(t *testing.T) { + expErr := errors.New("Replica flags are 'any-error,slave', expected 'slave'") + st := setupDbState() + st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") + st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave") + st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "any-error,slave") + err := st.state.IsOnline() + assert.Equal(t, expErr, err) +} + +func TestGetAddressReplicasSuccessfully(t *testing.T) { + st := setupDbState() + st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") + st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave") + st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave") + addr := st.state.ReplicasDbState.States[0].GetAddress() + assert.Equal(t, "6.7.8.9:1234", addr) + addr = st.state.ReplicasDbState.States[1].GetAddress() + assert.Equal(t, "6.7.8.10:3450", addr) +} + +func TestGetAddressReplicasNoIpPort(t *testing.T) { + st := setupDbState() + st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") + st.addReplicaFields("slave", "", "", "ok", "slave") + st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave") + addr := st.state.ReplicasDbState.States[0].GetAddress() + assert.Equal(t, "", addr) + addr = st.state.ReplicasDbState.States[1].GetAddress() + assert.Equal(t, "6.7.8.10:3450", addr) +} diff --git a/internal/sdlgoredis/sdlgoredis.go b/internal/sdlgoredis/sdlgoredis.go index 570dfa1..5f64960 100644 --- a/internal/sdlgoredis/sdlgoredis.go +++ b/internal/sdlgoredis/sdlgoredis.go @@ -24,8 +24,9 @@ package sdlgoredis import ( "errors" - "fmt" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v7" + "io" + "log" "os" "strconv" "strings" @@ -57,10 +58,13 @@ type Config struct { type DB struct { client RedisClient + sentinel RedisSentinelCreateCb subscribe SubscribeFn redisModules bool sCbMap *sharedCbMap ch intChannels + cfg Config + addr string } type Subscriber interface { @@ -92,6 +96,18 @@ type RedisClient interface { EvalSha(sha1 string, keys []string, args ...interface{}) *redis.Cmd ScriptExists(scripts ...string) *redis.BoolSliceCmd ScriptLoad(script string) *redis.StringCmd + Info(section ...string) *redis.StringCmd +} + +var dbLogger *log.Logger + +func init() { + dbLogger = log.New(os.Stdout, "database: ", log.LstdFlags|log.Lshortfile) + redis.SetLogger(dbLogger) +} + +func SetDbLogger(out io.Writer) { + dbLogger.SetOutput(out) } func checkResultAndError(result interface{}, err error) (bool, error) { @@ -127,9 +143,10 @@ func subscribeNotifications(client RedisClient, channels ...string) Subscriber { return client.Subscribe(channels...) } -func CreateDB(client RedisClient, subscribe SubscribeFn) *DB { +func CreateDB(client RedisClient, subscribe SubscribeFn, sentinelCreateCb RedisSentinelCreateCb, cfg Config, sentinelAddr string) *DB { db := DB{ client: client, + sentinel: sentinelCreateCb, subscribe: subscribe, redisModules: true, sCbMap: &sharedCbMap{cbMap: make(map[string]ChannelNotificationCb, 0)}, @@ -138,6 +155,8 @@ func CreateDB(client RedisClient, subscribe SubscribeFn) *DB { removeChannel: make(chan string), exit: make(chan bool), }, + cfg: cfg, + addr: sentinelAddr, } return &db @@ -145,7 +164,7 @@ func CreateDB(client RedisClient, subscribe SubscribeFn) *DB { func Create() []*DB { osimpl := osImpl{} - return ReadConfigAndCreateDbClients(osimpl, newRedisClient) + return ReadConfigAndCreateDbClients(osimpl, newRedisClient, subscribeNotifications, newRedisSentinel) } func readConfig(osI OS) Config { @@ -173,38 +192,48 @@ func (osImpl) Getenv(key string, defValue string) string { return val } -func ReadConfigAndCreateDbClients(osI OS, clientCreator RedisClientCreator) []*DB { +func ReadConfigAndCreateDbClients(osI OS, clientCreator RedisClientCreator, + subscribe SubscribeFn, + sentinelCreateCb RedisSentinelCreateCb) []*DB { cfg := readConfig(osI) - return createDbClients(cfg, clientCreator) + return createDbClients(cfg, clientCreator, subscribe, sentinelCreateCb) } -func createDbClients(cfg Config, clientCreator RedisClientCreator) []*DB { +func createDbClients(cfg Config, clientCreator RedisClientCreator, + subscribe SubscribeFn, + sentinelCreateCb RedisSentinelCreateCb) []*DB { if cfg.clusterAddrList == "" { - return []*DB{createLegacyDbClient(cfg, clientCreator)} + return []*DB{createLegacyDbClient(cfg, clientCreator, subscribe, sentinelCreateCb)} } dbs := []*DB{} addrList := strings.Split(cfg.clusterAddrList, ",") for _, addr := range addrList { - db := createDbClient(cfg, addr, clientCreator) + db := createDbClient(cfg, addr, clientCreator, subscribe, sentinelCreateCb) dbs = append(dbs, db) } return dbs } -func createLegacyDbClient(cfg Config, clientCreator RedisClientCreator) *DB { - return createDbClient(cfg, cfg.hostname, clientCreator) +func createLegacyDbClient(cfg Config, clientCreator RedisClientCreator, + subscribe SubscribeFn, + sentinelCreateCb RedisSentinelCreateCb) *DB { + return createDbClient(cfg, cfg.hostname, clientCreator, subscribe, sentinelCreateCb) } -func createDbClient(cfg Config, hostName string, clientCreator RedisClientCreator) *DB { +func createDbClient(cfg Config, hostName string, clientCreator RedisClientCreator, + subscribe SubscribeFn, + sentinelCreateCb RedisSentinelCreateCb) *DB { var client RedisClient + var db *DB if cfg.sentinelPort == "" { client = clientCreator(hostName, cfg.port, "", false) + db = CreateDB(client, subscribe, nil, cfg, hostName) } else { client = clientCreator(hostName, cfg.sentinelPort, cfg.masterName, true) + db = CreateDB(client, subscribe, sentinelCreateCb, cfg, hostName) } - db := CreateDB(client, subscribeNotifications) db.CheckCommands() return db } @@ -243,7 +272,7 @@ func (db *DB) CheckCommands() { } } } else { - fmt.Println(err) + dbLogger.Printf("SDL DB commands checking failure: %s\n", err) } } @@ -291,7 +320,7 @@ func (db *DB) SubscribeChannelDB(cb func(string, ...string), channelPrefix, even case exit := <-ch.exit: if exit { if err := sub.Close(); err != nil { - fmt.Println(err) + dbLogger.Printf("SDL DB channel closing failure: %s\n", err) } return } @@ -455,6 +484,66 @@ func (db *DB) PTTL(key string) (time.Duration, error) { return result, err } +func (db *DB) Info() (*DbInfo, error) { + var info DbInfo + resultStr, err := db.client.Info("all").Result() + result := strings.Split(strings.ReplaceAll(resultStr, "\r\n", "\n"), "\n") + readRedisInfoReplyFields(result, &info) + return &info, err +} + +func readRedisInfoReplyFields(input []string, info *DbInfo) { + for _, line := range input { + if idx := strings.Index(line, "role:"); idx != -1 { + roleStr := line[idx+len("role:"):] + if roleStr == "master" { + info.Fields.MasterRole = true + } + } else if idx := strings.Index(line, "connected_slaves:"); idx != -1 { + cntStr := line[idx+len("connected_slaves:"):] + if cnt, err := strconv.ParseUint(cntStr, 10, 32); err == nil { + info.Fields.ConnectedReplicaCnt = uint32(cnt) + } + } + } +} + +func (db *DB) State() (*DbState, error) { + if db.cfg.sentinelPort != "" { + //Establish connection to Redis sentinel. The reason why connection is done + //here instead of time of the SDL instance creation is that for the time being + //sentinel connection is needed only here to get state information and + //state information is needed only by 'sdlcli' hence it is not time critical + //and also we want to avoid opening unnecessary TCP connections towards Redis + //sentinel for every SDL instance. Now it is done only when 'sdlcli' is used. + sentinelClient := db.sentinel(&db.cfg, db.addr) + return sentinelClient.GetDbState() + } else { + var dbState DbState + info, err := db.Info() + if err != nil { + return &dbState, err + } + dbState = fillDbStateFromDbInfo(info) + return &dbState, err + } +} + +func fillDbStateFromDbInfo(info *DbInfo) DbState { + var dbState DbState + if info.Fields.MasterRole == true { + dbState = DbState{ + MasterDbState: MasterDbState{ + Fields: MasterDbStateFields{ + Role: "master", + Flags: "master", + }, + }, + } + } + return dbState +} + var luaRefresh = redis.NewScript(`if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("pexpire", KEYS[1], ARGV[2]) else return 0 end`) func (db *DB) PExpireIE(key string, data interface{}, expiration time.Duration) error { diff --git a/internal/sdlgoredis/sdlgoredis_test.go b/internal/sdlgoredis/sdlgoredis_test.go index 469e8c4..287115b 100644 --- a/internal/sdlgoredis/sdlgoredis_test.go +++ b/internal/sdlgoredis/sdlgoredis_test.go @@ -24,14 +24,13 @@ package sdlgoredis_test import ( "errors" - "strconv" - "testing" - "time" - "gerrit.o-ran-sc.org/r/ric-plt/sdlgo/internal/sdlgoredis" - "github.com/go-redis/redis" + "github.com/go-redis/redis/v7" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "strconv" + "testing" + "time" ) type clientMock struct { @@ -138,6 +137,23 @@ func (m *clientMock) ScriptLoad(script string) *redis.StringCmd { return m.Called(script).Get(0).(*redis.StringCmd) } +func (m *clientMock) Info(section ...string) *redis.StringCmd { + return m.Called(section).Get(0).(*redis.StringCmd) +} + +type MockRedisSentinel struct { + mock.Mock +} + +func (m *MockRedisSentinel) Master(name string) *redis.StringStringMapCmd { + a := m.Called(name) + return a.Get(0).(*redis.StringStringMapCmd) +} +func (m *MockRedisSentinel) Slaves(name string) *redis.SliceCmd { + a := m.Called(name) + return a.Get(0).(*redis.SliceCmd) +} + func setSubscribeNotifications() (*pubSubMock, sdlgoredis.SubscribeFn) { mock := new(pubSubMock) return mock, func(client sdlgoredis.RedisClient, channels ...string) sdlgoredis.Subscriber { @@ -150,16 +166,48 @@ func (m *MockOS) Getenv(key string, defValue string) string { return a.String(0) } -func setup(commandsExists bool) (*pubSubMock, *clientMock, *sdlgoredis.DB) { - mock := new(clientMock) - pubSubMock, subscribeNotifications := setSubscribeNotifications() - db := sdlgoredis.CreateDB(mock, subscribeNotifications) +type setupEv struct { + pubSubMock []*pubSubMock + rClient []*clientMock + rSentinel []*MockRedisSentinel + db []*sdlgoredis.DB +} + +func setupHaEnv(commandsExists bool) (*pubSubMock, *clientMock, *sdlgoredis.DB) { + psm, cm, _, db := setupHaEnvWithSentinels(commandsExists) + return psm, cm, db +} + +func setupHaEnvWithSentinels(commandsExists bool) (*pubSubMock, *clientMock, []*MockRedisSentinel, *sdlgoredis.DB) { + setupVals := setupEnv( + commandsExists, + "service-ricplt-dbaas-tcp-cluster-0.ricplt", + "6376", + "dbaasmaster", + "26376", + "", + "3", + ) + return setupVals.pubSubMock[0], setupVals.rClient[0], setupVals.rSentinel, setupVals.db[0] +} + +func setupSingleEnv(commandsExists bool) (*pubSubMock, *clientMock, *sdlgoredis.DB) { + setupVals := setupEnv( + commandsExists, + "service-ricplt-dbaas-tcp-cluster-0.ricplt", + "6376", "", "", "", "", + ) + return setupVals.pubSubMock[0], setupVals.rClient[0], setupVals.db[0] +} + +func setupEnv(commandsExists bool, host, port, msname, sntport, clsaddrlist, nodeCnt string) setupEv { + var ret setupEv dummyCommandInfo := redis.CommandInfo{ Name: "dummy", } - cmdResult := make(map[string]*redis.CommandInfo, 0) + cmdResult := make(map[string]*redis.CommandInfo, 0) if commandsExists { cmdResult = map[string]*redis.CommandInfo{ "setie": &dummyCommandInfo, @@ -175,45 +223,41 @@ func setup(commandsExists bool) (*pubSubMock, *clientMock, *sdlgoredis.DB) { } } - mock.On("Command").Return(redis.NewCommandsInfoCmdResult(cmdResult, nil)) - db.CheckCommands() - return pubSubMock, mock, db -} - -func setupEnv(host, port, msname, sntport, clsaddrlist string) ([]*clientMock, []*sdlgoredis.DB) { - var clmocks []*clientMock - - dummyCommandInfo := redis.CommandInfo{ - Name: "dummy", - } - cmdResult := make(map[string]*redis.CommandInfo, 0) - - cmdResult = map[string]*redis.CommandInfo{ - "dummy": &dummyCommandInfo, - } - osmock := new(MockOS) osmock.On("Getenv", "DBAAS_SERVICE_HOST", "localhost").Return(host) osmock.On("Getenv", "DBAAS_SERVICE_PORT", "6379").Return(port) osmock.On("Getenv", "DBAAS_MASTER_NAME", "").Return(msname) osmock.On("Getenv", "DBAAS_SERVICE_SENTINEL_PORT", "").Return(sntport) osmock.On("Getenv", "DBAAS_CLUSTER_ADDR_LIST", "").Return(clsaddrlist) + osmock.On("Getenv", "DBAAS_SERVICE_NODE_COUNT", "").Return(nodeCnt) + pubSubMock, subscribeNotifications := setSubscribeNotifications() + smock := new(MockRedisSentinel) + ret.rSentinel = append(ret.rSentinel, smock) clients := sdlgoredis.ReadConfigAndCreateDbClients( osmock, func(addr, port, clusterName string, isHa bool) sdlgoredis.RedisClient { clm := new(clientMock) clm.On("Command").Return(redis.NewCommandsInfoCmdResult(cmdResult, nil)) - clmocks = append(clmocks, clm) + ret.rClient = append(ret.rClient, clm) + ret.pubSubMock = append(ret.pubSubMock, pubSubMock) return clm }, + subscribeNotifications, + func(cfg *sdlgoredis.Config, addr string) *sdlgoredis.Sentinel { + s := &sdlgoredis.Sentinel{ + IredisSentinelClient: smock, + Cfg: cfg, + } + return s + }, ) - - return clmocks, clients + ret.db = clients + return ret } func TestCloseDbSuccessfully(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) r.On("Close").Return(nil) err := db.CloseDB() assert.Nil(t, err) @@ -221,7 +265,7 @@ func TestCloseDbSuccessfully(t *testing.T) { } func TestCloseDbFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) r.On("Close").Return(errors.New("Some error")) err := db.CloseDB() assert.NotNil(t, err) @@ -229,7 +273,7 @@ func TestCloseDbFailure(t *testing.T) { } func TestMSetSuccessfully(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKeysAndValues := []interface{}{"key1", "value1", "key2", 2} r.On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil)) err := db.MSet("key1", "value1", "key2", 2) @@ -238,7 +282,7 @@ func TestMSetSuccessfully(t *testing.T) { } func TestMSetFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKeysAndValues := []interface{}{"key1", "value1", "key2", 2} r.On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", errors.New("Some error"))) err := db.MSet("key1", "value1", "key2", 2) @@ -247,7 +291,7 @@ func TestMSetFailure(t *testing.T) { } func TestMSetMPubSuccessfully(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"MSETMPUB", 2, 2, "key1", "val1", "key2", "val2", "chan1", "event1", "chan2", "event2"} r.On("Do", expectedMessage).Return(redis.NewCmdResult("", nil)) @@ -257,7 +301,7 @@ func TestMSetMPubSuccessfully(t *testing.T) { } func TestMsetMPubFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"MSETMPUB", 2, 2, "key1", "val1", "key2", "val2", "chan1", "event1", "chan2", "event2"} r.On("Do", expectedMessage).Return(redis.NewCmdResult("", errors.New("Some error"))) @@ -267,7 +311,7 @@ func TestMsetMPubFailure(t *testing.T) { } func TestMSetMPubCommandMissing(t *testing.T) { - _, r, db := setup(false) + _, r, db := setupHaEnv(false) expectedMessage := []interface{}{"MSETMPUB", 2, 2, "key1", "val1", "key2", "val2", "chan1", "event1", "chan2", "event2"} r.AssertNotCalled(t, "Do", expectedMessage) @@ -278,7 +322,7 @@ func TestMSetMPubCommandMissing(t *testing.T) { } func TestMGetSuccessfully(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKeys := []string{"key1", "key2", "key3"} expectedResult := []interface{}{"val1", 2, nil} r.On("MGet", expectedKeys).Return(redis.NewSliceResult(expectedResult, nil)) @@ -289,7 +333,7 @@ func TestMGetSuccessfully(t *testing.T) { } func TestMGetFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKeys := []string{"key1", "key2", "key3"} expectedResult := []interface{}{nil} r.On("MGet", expectedKeys).Return(redis.NewSliceResult(expectedResult, @@ -301,7 +345,7 @@ func TestMGetFailure(t *testing.T) { } func TestDelMPubSuccessfully(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"DELMPUB", 2, 2, "key1", "key2", "chan1", "event1", "chan2", "event2"} r.On("Do", expectedMessage).Return(redis.NewCmdResult("", nil)) @@ -311,7 +355,7 @@ func TestDelMPubSuccessfully(t *testing.T) { } func TestDelMPubFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"DELMPUB", 2, 2, "key1", "key2", "chan1", "event1", "chan2", "event2"} r.On("Do", expectedMessage).Return(redis.NewCmdResult("", errors.New("Some error"))) @@ -321,7 +365,7 @@ func TestDelMPubFailure(t *testing.T) { } func TestDelMPubCommandMissing(t *testing.T) { - _, r, db := setup(false) + _, r, db := setupHaEnv(false) expectedMessage := []interface{}{"DELMPUB", 2, 2, "key1", "key2", "chan1", "event1", "chan2", "event2"} r.AssertNotCalled(t, "Do", expectedMessage) @@ -331,7 +375,7 @@ func TestDelMPubCommandMissing(t *testing.T) { } func TestDelSuccessfully(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKeys := []string{"key1", "key2"} r.On("Del", expectedKeys).Return(redis.NewIntResult(2, nil)) assert.Nil(t, db.Del([]string{"key1", "key2"})) @@ -339,7 +383,7 @@ func TestDelSuccessfully(t *testing.T) { } func TestDelFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKeys := []string{"key1", "key2"} r.On("Del", expectedKeys).Return(redis.NewIntResult(2, errors.New("Some error"))) assert.NotNil(t, db.Del([]string{"key1", "key2"})) @@ -347,7 +391,7 @@ func TestDelFailure(t *testing.T) { } func TestKeysSuccessfully(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedPattern := "pattern*" expectedResult := []string{"pattern1", "pattern2"} r.On("Keys", expectedPattern).Return(redis.NewStringSliceResult(expectedResult, nil)) @@ -358,7 +402,7 @@ func TestKeysSuccessfully(t *testing.T) { } func TestKeysFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedPattern := "pattern*" expectedResult := []string{} r.On("Keys", expectedPattern).Return(redis.NewStringSliceResult(expectedResult, @@ -369,7 +413,7 @@ func TestKeysFailure(t *testing.T) { } func TestSetIEKeyExists(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"SETIE", "key", "newdata", "olddata"} r.On("Do", expectedMessage).Return(redis.NewCmdResult("OK", nil)) result, err := db.SetIE("key", "olddata", "newdata") @@ -379,7 +423,7 @@ func TestSetIEKeyExists(t *testing.T) { } func TestSetIEKeyDoesntExists(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"SETIE", "key", "newdata", "olddata"} r.On("Do", expectedMessage).Return(redis.NewCmdResult(nil, nil)) result, err := db.SetIE("key", "olddata", "newdata") @@ -389,7 +433,7 @@ func TestSetIEKeyDoesntExists(t *testing.T) { } func TestSetIEFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"SETIE", "key", "newdata", "olddata"} r.On("Do", expectedMessage).Return(redis.NewCmdResult(nil, errors.New("Some error"))) result, err := db.SetIE("key", "olddata", "newdata") @@ -399,7 +443,7 @@ func TestSetIEFailure(t *testing.T) { } func TestSetIECommandMissing(t *testing.T) { - _, r, db := setup(false) + _, r, db := setupHaEnv(false) expectedMessage := []interface{}{"SETIE", "key", "newdata", "olddata"} r.AssertNotCalled(t, "Do", expectedMessage) result, err := db.SetIE("key", "olddata", "newdata") @@ -409,7 +453,7 @@ func TestSetIECommandMissing(t *testing.T) { } func TestSetIEPubKeyExists(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"SETIEMPUB", "key", "newdata", "olddata", "channel", "message"} r.On("Do", expectedMessage).Return(redis.NewCmdResult("OK", nil)) result, err := db.SetIEPub([]string{"channel", "message"}, "key", "olddata", "newdata") @@ -419,7 +463,7 @@ func TestSetIEPubKeyExists(t *testing.T) { } func TestSetIEPubKeyDoesntExists(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"SETIEMPUB", "key", "newdata", "olddata", "channel", "message"} r.On("Do", expectedMessage).Return(redis.NewCmdResult(nil, nil)) result, err := db.SetIEPub([]string{"channel", "message"}, "key", "olddata", "newdata") @@ -429,7 +473,7 @@ func TestSetIEPubKeyDoesntExists(t *testing.T) { } func TestSetIEPubFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"SETIEMPUB", "key", "newdata", "olddata", "channel", "message"} r.On("Do", expectedMessage).Return(redis.NewCmdResult(nil, errors.New("Some error"))) result, err := db.SetIEPub([]string{"channel", "message"}, "key", "olddata", "newdata") @@ -439,7 +483,7 @@ func TestSetIEPubFailure(t *testing.T) { } func TestSetIEPubCommandMissing(t *testing.T) { - _, r, db := setup(false) + _, r, db := setupHaEnv(false) expectedMessage := []interface{}{"SETIEMPUB", "key", "newdata", "olddata", "channel", "message"} r.AssertNotCalled(t, "Do", expectedMessage) result, err := db.SetIEPub([]string{"channel", "message"}, "key", "olddata", "newdata") @@ -449,7 +493,7 @@ func TestSetIEPubCommandMissing(t *testing.T) { } func TestSetNXPubKeyDoesntExist(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"SETNXMPUB", "key", "data", "channel", "message"} r.On("Do", expectedMessage).Return(redis.NewCmdResult("OK", nil)) result, err := db.SetNXPub([]string{"channel", "message"}, "key", "data") @@ -459,7 +503,7 @@ func TestSetNXPubKeyDoesntExist(t *testing.T) { } func TestSetNXPubKeyExists(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"SETNXMPUB", "key", "data", "channel", "message"} r.On("Do", expectedMessage).Return(redis.NewCmdResult(nil, nil)) result, err := db.SetNXPub([]string{"channel", "message"}, "key", "data") @@ -469,7 +513,7 @@ func TestSetNXPubKeyExists(t *testing.T) { } func TestSetNXPubFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"SETNXMPUB", "key", "data", "channel", "message"} r.On("Do", expectedMessage).Return(redis.NewCmdResult(nil, errors.New("Some error"))) result, err := db.SetNXPub([]string{"channel", "message"}, "key", "data") @@ -479,7 +523,7 @@ func TestSetNXPubFailure(t *testing.T) { } func TestSetNXPubCommandMissing(t *testing.T) { - _, r, db := setup(false) + _, r, db := setupHaEnv(false) expectedMessage := []interface{}{"SETNXMPUB", "key", "data", "channel", "message"} r.AssertNotCalled(t, "Do", expectedMessage) result, err := db.SetNXPub([]string{"channel", "message"}, "key", "data") @@ -489,7 +533,7 @@ func TestSetNXPubCommandMissing(t *testing.T) { } func TestSetNXSuccessfully(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedData := "data" r.On("SetNX", expectedKey, expectedData, time.Duration(0)).Return(redis.NewBoolResult(true, nil)) @@ -500,7 +544,7 @@ func TestSetNXSuccessfully(t *testing.T) { } func TestSetNXUnsuccessfully(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedData := "data" r.On("SetNX", expectedKey, expectedData, time.Duration(0)).Return(redis.NewBoolResult(false, nil)) @@ -511,7 +555,7 @@ func TestSetNXUnsuccessfully(t *testing.T) { } func TestSetNXFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedData := "data" r.On("SetNX", expectedKey, expectedData, time.Duration(0)). @@ -523,7 +567,7 @@ func TestSetNXFailure(t *testing.T) { } func TestDelIEPubKeyDoesntExist(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"DELIEMPUB", "key", "data", "channel", "message"} r.On("Do", expectedMessage).Return(redis.NewCmdResult(int64(0), nil)) result, err := db.DelIEPub([]string{"channel", "message"}, "key", "data") @@ -533,7 +577,7 @@ func TestDelIEPubKeyDoesntExist(t *testing.T) { } func TestDelIEPubKeyExists(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"DELIEMPUB", "key", "data", "channel", "message"} r.On("Do", expectedMessage).Return(redis.NewCmdResult(int64(1), nil)) result, err := db.DelIEPub([]string{"channel", "message"}, "key", "data") @@ -543,7 +587,7 @@ func TestDelIEPubKeyExists(t *testing.T) { } func TestDelIEPubKeyExistsIntTypeRedisValue(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"DELIEMPUB", "key", "data", "channel", "message"} r.On("Do", expectedMessage).Return(redis.NewCmdResult(1, nil)) result, err := db.DelIEPub([]string{"channel", "message"}, "key", "data") @@ -553,7 +597,7 @@ func TestDelIEPubKeyExistsIntTypeRedisValue(t *testing.T) { } func TestDelIEPubFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"DELIEMPUB", "key", "data", "channel", "message"} r.On("Do", expectedMessage).Return(redis.NewCmdResult(int64(0), errors.New("Some error"))) result, err := db.DelIEPub([]string{"channel", "message"}, "key", "data") @@ -563,7 +607,7 @@ func TestDelIEPubFailure(t *testing.T) { } func TestDelIEPubCommandMissing(t *testing.T) { - _, r, db := setup(false) + _, r, db := setupHaEnv(false) expectedMessage := []interface{}{"DELIEMPUB", "key", "data", "channel", "message"} r.AssertNotCalled(t, "Do", expectedMessage) result, err := db.DelIEPub([]string{"channel", "message"}, "key", "data") @@ -573,7 +617,7 @@ func TestDelIEPubCommandMissing(t *testing.T) { } func TestDelIEKeyDoesntExist(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"DELIE", "key", "data"} r.On("Do", expectedMessage).Return(redis.NewCmdResult(int64(0), nil)) result, err := db.DelIE("key", "data") @@ -583,7 +627,7 @@ func TestDelIEKeyDoesntExist(t *testing.T) { } func TestDelIEKeyExists(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"DELIE", "key", "data"} r.On("Do", expectedMessage).Return(redis.NewCmdResult(int64(1), nil)) result, err := db.DelIE("key", "data") @@ -593,7 +637,7 @@ func TestDelIEKeyExists(t *testing.T) { } func TestDelIEKeyExistsIntTypeRedisValue(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"DELIE", "key", "data"} r.On("Do", expectedMessage).Return(redis.NewCmdResult(1, nil)) result, err := db.DelIE("key", "data") @@ -603,7 +647,7 @@ func TestDelIEKeyExistsIntTypeRedisValue(t *testing.T) { } func TestDelIEFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedMessage := []interface{}{"DELIE", "key", "data"} r.On("Do", expectedMessage).Return(redis.NewCmdResult(int64(0), errors.New("Some error"))) result, err := db.DelIE("key", "data") @@ -613,7 +657,7 @@ func TestDelIEFailure(t *testing.T) { } func TestDelIECommandMissing(t *testing.T) { - _, r, db := setup(false) + _, r, db := setupHaEnv(false) expectedMessage := []interface{}{"DELIE", "key", "data"} r.AssertNotCalled(t, "Do", expectedMessage) result, err := db.DelIE("key", "data") @@ -623,7 +667,7 @@ func TestDelIECommandMissing(t *testing.T) { } func TestSAddSuccessfully(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedData := []interface{}{"data", 2} r.On("SAdd", expectedKey, expectedData).Return(redis.NewIntResult(2, nil)) @@ -632,7 +676,7 @@ func TestSAddSuccessfully(t *testing.T) { } func TestSAddFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedData := []interface{}{"data", 2} r.On("SAdd", expectedKey, expectedData).Return(redis.NewIntResult(2, errors.New("Some error"))) @@ -641,7 +685,7 @@ func TestSAddFailure(t *testing.T) { } func TestSRemSuccessfully(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedData := []interface{}{"data", 2} r.On("SRem", expectedKey, expectedData).Return(redis.NewIntResult(2, nil)) @@ -650,7 +694,7 @@ func TestSRemSuccessfully(t *testing.T) { } func TestSRemFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedData := []interface{}{"data", 2} r.On("SRem", expectedKey, expectedData).Return(redis.NewIntResult(2, errors.New("Some error"))) @@ -659,7 +703,7 @@ func TestSRemFailure(t *testing.T) { } func TestSMembersSuccessfully(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedResult := []string{"member1", "member2"} r.On("SMembers", expectedKey).Return(redis.NewStringSliceResult(expectedResult, nil)) @@ -670,7 +714,7 @@ func TestSMembersSuccessfully(t *testing.T) { } func TestSMembersFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedResult := []string{"member1", "member2"} r.On("SMembers", expectedKey).Return(redis.NewStringSliceResult(expectedResult, @@ -682,7 +726,7 @@ func TestSMembersFailure(t *testing.T) { } func TestSIsMemberIsMember(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedData := "data" r.On("SIsMember", expectedKey, expectedData).Return(redis.NewBoolResult(true, nil)) @@ -693,7 +737,7 @@ func TestSIsMemberIsMember(t *testing.T) { } func TestSIsMemberIsNotMember(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedData := "data" r.On("SIsMember", expectedKey, expectedData).Return(redis.NewBoolResult(false, nil)) @@ -704,7 +748,7 @@ func TestSIsMemberIsNotMember(t *testing.T) { } func TestSIsMemberFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedData := "data" r.On("SIsMember", expectedKey, expectedData). @@ -716,7 +760,7 @@ func TestSIsMemberFailure(t *testing.T) { } func TestSCardSuccessfully(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" r.On("SCard", expectedKey).Return(redis.NewIntResult(1, nil)) result, err := db.SCard("key") @@ -726,7 +770,7 @@ func TestSCardSuccessfully(t *testing.T) { } func TestSCardFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" r.On("SCard", expectedKey).Return(redis.NewIntResult(1, errors.New("Some error"))) result, err := db.SCard("key") @@ -736,7 +780,7 @@ func TestSCardFailure(t *testing.T) { } func TestSubscribeChannelDBSubscribeRXUnsubscribe(t *testing.T) { - ps, r, db := setup(true) + ps, r, db := setupHaEnv(true) ch := make(chan *redis.Message) msg := redis.Message{ Channel: "{prefix}channel", @@ -762,7 +806,7 @@ func TestSubscribeChannelDBSubscribeRXUnsubscribe(t *testing.T) { } func TestSubscribeChannelDBSubscribeTwoUnsubscribeOne(t *testing.T) { - ps, r, db := setup(true) + ps, r, db := setupHaEnv(true) ch := make(chan *redis.Message) msg1 := redis.Message{ Channel: "{prefix}channel1", @@ -805,7 +849,7 @@ func TestSubscribeChannelDBSubscribeTwoUnsubscribeOne(t *testing.T) { } func TestSubscribeChannelReDBSubscribeAfterUnsubscribe(t *testing.T) { - ps, r, db := setup(true) + ps, r, db := setupHaEnv(true) ch := make(chan *redis.Message) msg := redis.Message{ Channel: "{prefix}channel", @@ -841,7 +885,7 @@ func TestSubscribeChannelReDBSubscribeAfterUnsubscribe(t *testing.T) { } func TestPTTLSuccessfully(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedResult := time.Duration(1) r.On("PTTL", expectedKey).Return(redis.NewDurationResult(expectedResult, @@ -853,7 +897,7 @@ func TestPTTLSuccessfully(t *testing.T) { } func TestPTTLFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedResult := time.Duration(1) r.On("PTTL", expectedKey).Return(redis.NewDurationResult(expectedResult, @@ -865,7 +909,7 @@ func TestPTTLFailure(t *testing.T) { } func TestPExpireIESuccessfully(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedData := "data" expectedDuration := strconv.FormatInt(int64(10000), 10) @@ -879,7 +923,7 @@ func TestPExpireIESuccessfully(t *testing.T) { } func TestPExpireIEFailure(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedData := "data" expectedDuration := strconv.FormatInt(int64(10000), 10) @@ -893,7 +937,7 @@ func TestPExpireIEFailure(t *testing.T) { } func TestPExpireIELockNotHeld(t *testing.T) { - _, r, db := setup(true) + _, r, db := setupHaEnv(true) expectedKey := "key" expectedData := "data" expectedDuration := strconv.FormatInt(int64(10000), 10) @@ -907,73 +951,330 @@ func TestPExpireIELockNotHeld(t *testing.T) { } func TestClientStandaloneRedisLegacyEnv(t *testing.T) { - rcls, dbs := setupEnv( - "service-ricplt-dbaas-tcp-cluster-0.ricplt", "6376", "", "", "", + setupVals := setupEnv( + true, + "service-ricplt-dbaas-tcp-cluster-0.ricplt", "6376", "", "", "", "", ) - assert.Equal(t, 1, len(rcls)) - assert.Equal(t, 1, len(dbs)) + assert.Equal(t, 1, len(setupVals.rClient)) + assert.Equal(t, 1, len(setupVals.db)) expectedKeysAndValues := []interface{}{"key1", "value1"} - rcls[0].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil)) - err := dbs[0].MSet("key1", "value1") + setupVals.rClient[0].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil)) + err := setupVals.db[0].MSet("key1", "value1") assert.Nil(t, err) - rcls[0].AssertExpectations(t) + setupVals.rClient[0].AssertExpectations(t) } func TestClientSentinelRedisLegacyEnv(t *testing.T) { - rcls, dbs := setupEnv( - "service-ricplt-dbaas-tcp-cluster-0.ricplt", "6376", "dbaasmaster", "26376", "", + setupVals := setupEnv( + true, + "service-ricplt-dbaas-tcp-cluster-0.ricplt", "6376", "dbaasmaster", "26376", "", "3", ) - assert.Equal(t, 1, len(rcls)) - assert.Equal(t, 1, len(dbs)) + assert.Equal(t, 1, len(setupVals.rClient)) + assert.Equal(t, 1, len(setupVals.db)) expectedKeysAndValues := []interface{}{"key1", "value1"} - rcls[0].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil)) - err := dbs[0].MSet("key1", "value1") + setupVals.rClient[0].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil)) + err := setupVals.db[0].MSet("key1", "value1") assert.Nil(t, err) - rcls[0].AssertExpectations(t) + setupVals.rClient[0].AssertExpectations(t) } func TestClientTwoStandaloneRedisEnvs(t *testing.T) { - rcls, dbs := setupEnv( + setupVals := setupEnv( + true, "service-ricplt-dbaas-tcp-cluster-0.ricplt", "6376", "", "", - "service-ricplt-dbaas-tcp-cluster-0.ricplt,service-ricplt-dbaas-tcp-cluster-1.ricplt", + "service-ricplt-dbaas-tcp-cluster-0.ricplt,service-ricplt-dbaas-tcp-cluster-1.ricplt", "", ) - assert.Equal(t, 2, len(rcls)) - assert.Equal(t, 2, len(dbs)) + assert.Equal(t, 2, len(setupVals.rClient)) + assert.Equal(t, 2, len(setupVals.db)) expectedKeysAndValues := []interface{}{"key1", "value1"} - rcls[0].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil)) - err := dbs[0].MSet("key1", "value1") + setupVals.rClient[0].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil)) + err := setupVals.db[0].MSet("key1", "value1") assert.Nil(t, err) - rcls[0].AssertExpectations(t) + setupVals.rClient[0].AssertExpectations(t) expectedKeysAndValues = []interface{}{"key2", "value2"} - rcls[1].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil)) - err = dbs[1].MSet("key2", "value2") + setupVals.rClient[1].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil)) + err = setupVals.db[1].MSet("key2", "value2") assert.Nil(t, err) - rcls[0].AssertExpectations(t) - rcls[1].AssertExpectations(t) + setupVals.rClient[0].AssertExpectations(t) + setupVals.rClient[1].AssertExpectations(t) } func TestClientTwoSentinelRedisEnvs(t *testing.T) { - rcls, dbs := setupEnv( + setupVals := setupEnv( + true, "service-ricplt-dbaas-tcp-cluster-0.ricplt", "6376", "dbaasmaster", "26376", - "service-ricplt-dbaas-tcp-cluster-0.ricplt,service-ricplt-dbaas-tcp-cluster-1.ricplt", + "service-ricplt-dbaas-tcp-cluster-0.ricplt,service-ricplt-dbaas-tcp-cluster-1.ricplt", "3", ) - assert.Equal(t, 2, len(rcls)) - assert.Equal(t, 2, len(dbs)) + assert.Equal(t, 2, len(setupVals.rClient)) + assert.Equal(t, 2, len(setupVals.db)) expectedKeysAndValues := []interface{}{"key1", "value1"} - rcls[0].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil)) - err := dbs[0].MSet("key1", "value1") + setupVals.rClient[0].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil)) + err := setupVals.db[0].MSet("key1", "value1") assert.Nil(t, err) - rcls[0].AssertExpectations(t) + setupVals.rClient[0].AssertExpectations(t) expectedKeysAndValues = []interface{}{"key2", "value2"} - rcls[1].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil)) - err = dbs[1].MSet("key2", "value2") + setupVals.rClient[1].On("MSet", expectedKeysAndValues).Return(redis.NewStatusResult("OK", nil)) + err = setupVals.db[1].MSet("key2", "value2") + assert.Nil(t, err) + setupVals.rClient[0].AssertExpectations(t) + setupVals.rClient[1].AssertExpectations(t) +} + +func TestInfoOfMasterRedisWithTwoSlavesSuccessfully(t *testing.T) { + _, r, db := setupHaEnv(true) + redisInfo := "# Replication\r\n" + + "role:master\r\n" + + "connected_slaves:2\r\n" + + "min_slaves_good_slaves:2\r\n" + + "slave0:ip=1.2.3.4,port=6379,state=online,offset=100200300,lag=0\r\n" + + "slave1:ip=5.6.7.8,port=6379,state=online,offset=100200300,lag=0\r\n" + expInfo := &sdlgoredis.DbInfo{ + Fields: sdlgoredis.DbInfoFields{ + MasterRole: true, + ConnectedReplicaCnt: 2, + }, + } + + r.On("Info", []string{"all"}).Return(redis.NewStringResult(redisInfo, nil)) + info, err := db.Info() + assert.Nil(t, err) + assert.Equal(t, expInfo, info) + r.AssertExpectations(t) +} + +func TestInfoOfMasterRedisWithOneSlaveOnlineAndOtherSlaveNotOnlineSuccessfully(t *testing.T) { + _, r, db := setupHaEnv(true) + redisInfo := "# Replication\r\n" + + "role:master\r\n" + + "connected_slaves:1\r\n" + + "min_slaves_good_slaves:2\r\n" + + "slave0:ip=1.2.3.4,port=6379,state=online,offset=100200300,lag=0\r\n" + + "slave1:ip=5.6.7.8,port=6379,state=wait_bgsave,offset=100200300,lag=0\r\n" + expInfo := &sdlgoredis.DbInfo{ + Fields: sdlgoredis.DbInfoFields{ + MasterRole: true, + ConnectedReplicaCnt: 1, + }, + } + + r.On("Info", []string{"all"}).Return(redis.NewStringResult(redisInfo, nil)) + info, err := db.Info() + assert.Nil(t, err) + assert.Equal(t, expInfo, info) + r.AssertExpectations(t) +} + +func TestInfoOfStandaloneMasterRedisSuccessfully(t *testing.T) { + _, r, db := setupHaEnv(true) + redisInfo := "# Replication\r\n" + + "role:master\r\n" + + "connected_slaves:0\r\n" + + "min_slaves_good_slaves:0\r\n" + expInfo := &sdlgoredis.DbInfo{ + Fields: sdlgoredis.DbInfoFields{ + MasterRole: true, + ConnectedReplicaCnt: 0, + }, + } + + r.On("Info", []string{"all"}).Return(redis.NewStringResult(redisInfo, nil)) + info, err := db.Info() + assert.Nil(t, err) + assert.Equal(t, expInfo, info) + r.AssertExpectations(t) +} + +func TestInfoWithGibberishContentSuccessfully(t *testing.T) { + _, r, db := setupHaEnv(true) + redisInfo := "!#¤%&?+?´-\r\n" + expInfo := &sdlgoredis.DbInfo{} + + r.On("Info", []string{"all"}).Return(redis.NewStringResult(redisInfo, nil)) + info, err := db.Info() + assert.Nil(t, err) + assert.Equal(t, expInfo, info) + r.AssertExpectations(t) +} + +func TestInfoWithEmptyContentSuccessfully(t *testing.T) { + _, r, db := setupHaEnv(true) + var redisInfo string + expInfo := &sdlgoredis.DbInfo{ + Fields: sdlgoredis.DbInfoFields{ + MasterRole: false, + }, + } + + r.On("Info", []string{"all"}).Return(redis.NewStringResult(redisInfo, nil)) + info, err := db.Info() + assert.Nil(t, err) + assert.Equal(t, expInfo, info) + r.AssertExpectations(t) +} + +func TestStateWithMasterAndTwoSlaveRedisSuccessfully(t *testing.T) { + _, r, s, db := setupHaEnvWithSentinels(true) + redisMasterState := map[string]string{ + "role-reported": "master", + } + redisSlavesState := make([]interface{}, 2) + redisSlavesState[0] = []interface{}{ + "role-reported", "slave", + "ip", "10.20.30.40", + "port", "6379", + "flags", "slave", + "master-link-status", "up", + } + redisSlavesState[1] = []interface{}{ + "master-link-status", "up", + "ip", "10.20.30.50", + "flags", "slave", + "port", "30000", + "role-reported", "slave", + } + + expState := &sdlgoredis.DbState{ + MasterDbState: sdlgoredis.MasterDbState{ + Fields: sdlgoredis.MasterDbStateFields{ + Role: "master", + }, + }, + ReplicasDbState: &sdlgoredis.ReplicasDbState{ + States: []*sdlgoredis.ReplicaDbState{ + &sdlgoredis.ReplicaDbState{ + Fields: sdlgoredis.ReplicaDbStateFields{ + Role: "slave", + Ip: "10.20.30.40", + Port: "6379", + MasterLinkStatus: "up", + Flags: "slave", + }, + }, + &sdlgoredis.ReplicaDbState{ + Fields: sdlgoredis.ReplicaDbStateFields{ + Role: "slave", + Ip: "10.20.30.50", + Port: "30000", + MasterLinkStatus: "up", + Flags: "slave", + }, + }, + }, + }, + } + + s[0].On("Master", "dbaasmaster").Return(redis.NewStringStringMapResult(redisMasterState, nil)) + s[0].On("Slaves", "dbaasmaster").Return(redis.NewSliceResult(redisSlavesState, nil)) + ret, err := db.State() + assert.Nil(t, err) + assert.Equal(t, expState, ret) + r.AssertExpectations(t) +} + +func TestStateWithMasterAndOneSlaveRedisFailureInMasterRedisCall(t *testing.T) { + _, r, s, db := setupHaEnvWithSentinels(true) + redisMasterState := map[string]string{} + redisSlavesState := make([]interface{}, 1) + redisSlavesState[0] = []interface{}{ + "role-reported", "slave", + "ip", "10.20.30.40", + "port", "6379", + "flags", "slave", + "master-link-status", "up", + } + + expState := &sdlgoredis.DbState{ + MasterDbState: sdlgoredis.MasterDbState{ + Err: errors.New("Some error"), + }, + ReplicasDbState: &sdlgoredis.ReplicasDbState{ + States: []*sdlgoredis.ReplicaDbState{ + &sdlgoredis.ReplicaDbState{ + Fields: sdlgoredis.ReplicaDbStateFields{ + Role: "slave", + Ip: "10.20.30.40", + Port: "6379", + MasterLinkStatus: "up", + Flags: "slave", + }, + }, + }, + }, + } + + s[0].On("Master", "dbaasmaster").Return(redis.NewStringStringMapResult(redisMasterState, errors.New("Some error"))) + s[0].On("Slaves", "dbaasmaster").Return(redis.NewSliceResult(redisSlavesState, nil)) + ret, err := db.State() + assert.NotNil(t, err) + assert.Equal(t, expState, ret) + r.AssertExpectations(t) +} + +func TestStateWithMasterAndOneSlaveRedisFailureInSlavesRedisCall(t *testing.T) { + _, r, s, db := setupHaEnvWithSentinels(true) + redisMasterState := map[string]string{ + "role-reported": "master", + } + redisSlavesState := make([]interface{}, 1) + redisSlavesState[0] = []interface{}{} + + expState := &sdlgoredis.DbState{ + MasterDbState: sdlgoredis.MasterDbState{ + Fields: sdlgoredis.MasterDbStateFields{ + Role: "master", + }, + }, + ReplicasDbState: &sdlgoredis.ReplicasDbState{ + Err: errors.New("Some error"), + States: []*sdlgoredis.ReplicaDbState{}, + }, + } + + s[0].On("Master", "dbaasmaster").Return(redis.NewStringStringMapResult(redisMasterState, nil)) + s[0].On("Slaves", "dbaasmaster").Return(redis.NewSliceResult(redisSlavesState, errors.New("Some error"))) + ret, err := db.State() + assert.NotNil(t, err) + assert.Equal(t, expState, ret) + r.AssertExpectations(t) +} + +func TestStateWithSingleMasterRedisSuccessfully(t *testing.T) { + _, r, db := setupSingleEnv(true) + redisInfo := "# Replication\r\n" + + "role:master\r\n" + + "connected_slaves:0\r\n" + + "min_slaves_good_slaves:0\r\n" + + expState := &sdlgoredis.DbState{ + MasterDbState: sdlgoredis.MasterDbState{ + Fields: sdlgoredis.MasterDbStateFields{ + Role: "master", + Flags: "master", + }, + }, + } + + r.On("Info", []string{"all"}).Return(redis.NewStringResult(redisInfo, nil)) + ret, err := db.State() assert.Nil(t, err) - rcls[0].AssertExpectations(t) - rcls[1].AssertExpectations(t) + assert.Equal(t, expState, ret) + r.AssertExpectations(t) +} + +func TestStateWithSingleMasterRedisFailureInInfoCall(t *testing.T) { + _, r, db := setupSingleEnv(true) + redisInfo := "" + expState := &sdlgoredis.DbState{} + + r.On("Info", []string{"all"}).Return(redis.NewStringResult(redisInfo, errors.New("Some error"))) + ret, err := db.State() + assert.NotNil(t, err) + assert.Equal(t, expState, ret) + r.AssertExpectations(t) } diff --git a/internal/sdlgoredis/sdlgosentinel.go b/internal/sdlgoredis/sdlgosentinel.go new file mode 100644 index 0000000..7af0410 --- /dev/null +++ b/internal/sdlgoredis/sdlgosentinel.go @@ -0,0 +1,111 @@ +/* + Copyright (c) 2021 AT&T Intellectual Property. + Copyright (c) 2018-2021 Nokia. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* + * This source code is part of the near-RT RIC (RAN Intelligent Controller) + * platform project (RICP). + */ + +package sdlgoredis + +import ( + "github.com/go-redis/redis/v7" +) + +type Sentinel struct { + IredisSentinelClient + Cfg *Config +} + +type IredisSentinelClient interface { + Master(name string) *redis.StringStringMapCmd + Slaves(name string) *redis.SliceCmd +} + +type RedisSentinelCreateCb func(cfg *Config, addr string) *Sentinel + +func newRedisSentinel(cfg *Config, addr string) *Sentinel { + redisAddress := addr + ":" + cfg.sentinelPort + return &Sentinel{ + IredisSentinelClient: redis.NewSentinelClient(&redis.Options{ + Addr: redisAddress, + Password: "", // no password set + DB: 0, // use default DB + PoolSize: 20, + MaxRetries: 2, + }), + Cfg: cfg, + } +} + +func (s *Sentinel) GetDbState() (*DbState, error) { + state := new(DbState) + mState, mErr := s.getMasterDbState() + rState, rErr := s.getReplicasState() + state.MasterDbState = *mState + state.ReplicasDbState = rState + if mErr == nil { + return state, rErr + } + return state, mErr +} + +func (s *Sentinel) getMasterDbState() (*MasterDbState, error) { + state := new(MasterDbState) + redisVal, redisErr := s.Master(s.Cfg.masterName).Result() + if redisErr == nil { + state.Fields.Ip = redisVal["ip"] + state.Fields.Port = redisVal["port"] + state.Fields.Flags = redisVal["flags"] + state.Fields.Role = redisVal["role-reported"] + } + state.Err = redisErr + return state, redisErr +} + +func (s *Sentinel) getReplicasState() (*ReplicasDbState, error) { + states := new(ReplicasDbState) + states.States = make([]*ReplicaDbState, 0) + + redisVal, redisErr := s.Slaves(s.Cfg.masterName).Result() + if redisErr == nil { + for _, redisSlave := range redisVal { + replicaState := readReplicaState(redisSlave.([]interface{})) + states.States = append(states.States, replicaState) + } + } + states.Err = redisErr + return states, redisErr +} + +func readReplicaState(redisSlaves []interface{}) *ReplicaDbState { + state := new(ReplicaDbState) + for i := 0; i < len(redisSlaves); i += 2 { + if redisSlaves[i].(string) == "ip" { + state.Fields.Ip = redisSlaves[i+1].(string) + } else if redisSlaves[i].(string) == "port" { + state.Fields.Port = redisSlaves[i+1].(string) + } else if redisSlaves[i].(string) == "flags" { + state.Fields.Flags = redisSlaves[i+1].(string) + } else if redisSlaves[i].(string) == "role-reported" { + state.Fields.Role = redisSlaves[i+1].(string) + } else if redisSlaves[i].(string) == "master-link-status" { + state.Fields.MasterLinkStatus = redisSlaves[i+1].(string) + } + } + return state +} -- 2.16.6