From 7c256b622c8fd065e91a7e289937d6e692a7eb1d Mon Sep 17 00:00:00 2001 From: Timo Tietavainen Date: Sun, 7 Nov 2021 22:25:46 +0200 Subject: [PATCH] Sentinel check to SDL CLI 'healthcheck' -command In addition to the existing Redis master and slave healthiness validation, add Redis sentinel healthiness validation to be done when SDL CLI 'healthcheck' -command is ran. Validation is done by calling Redis client's 'Sentinels' API call and parsing 'flags' field from its output. In SDL CLI sentinel healthiness status is shown only if status is not ok. This is done not to show too much information in CLI interface for users. Issue-Id: RIC-113 Signed-off-by: Timo Tietavainen Change-Id: I8960ec9813e06aa88217c29bac56707b38052ae7 --- internal/cli/healthcheck.go | 9 +++ internal/cli/healthcheck_test.go | 75 +++++++++++++++++--- internal/sdlgoredis/dbstate.go | 56 ++++++++++++++- internal/sdlgoredis/dbstate_test.go | 79 +++++++++++++++++++++ internal/sdlgoredis/sdlgoredis_test.go | 126 +++++++++++++++++++++++++++++++++ internal/sdlgoredis/sdlgosentinel.go | 39 +++++++++- 6 files changed, 372 insertions(+), 12 deletions(-) diff --git a/internal/cli/healthcheck.go b/internal/cli/healthcheck.go index 94467a9..aca1fda 100644 --- a/internal/cli/healthcheck.go +++ b/internal/cli/healthcheck.go @@ -96,6 +96,15 @@ func writeStateResults(dbStates []sdlgoredis.DbState) string { } } } + if dbState.SentinelsDbState != nil { + for k, sInfo := range dbState.SentinelsDbState.States { + err := sInfo.IsOnline() + if err != nil { + str = str + fmt.Sprintf(" Sentinel #%d (%s): NOK\n", (k+1), sInfo.GetAddress()) + str = str + fmt.Sprintf(" %s\n", err.Error()) + } + } + } } if anyErr == nil { str = fmt.Sprintf("Overall status: OK\n\n") + str diff --git a/internal/cli/healthcheck_test.go b/internal/cli/healthcheck_test.go index be98aed..e1de269 100644 --- a/internal/cli/healthcheck_test.go +++ b/internal/cli/healthcheck_test.go @@ -40,7 +40,7 @@ type healthCheckMocks struct { dbState sdlgoredis.DbState } -func setupHcMockMasterDb(ip, port string, replicas uint32) { +func setupHcMockMasterDb(ip, port string) { hcMocks = new(healthCheckMocks) hcMocks.dbState.MasterDbState.Fields.Role = "master" hcMocks.dbState.MasterDbState.Fields.Ip = ip @@ -48,7 +48,7 @@ func setupHcMockMasterDb(ip, port string, replicas uint32) { hcMocks.dbState.MasterDbState.Fields.Flags = "master" } -func setupHcMockReplicaDb() { +func setupHcMockReplicaDb(ip, port string) { hcMocks = new(healthCheckMocks) hcMocks.dbState.ReplicasDbState = new(sdlgoredis.ReplicasDbState) hcMocks.dbState.ReplicasDbState.States = []*sdlgoredis.ReplicaDbState{ @@ -60,6 +60,19 @@ func setupHcMockReplicaDb() { } } +func setupHcMockSentinelDb(ip, port string) { + hcMocks = new(healthCheckMocks) + hcMocks.dbState.SentinelsDbState = new(sdlgoredis.SentinelsDbState) + hcMocks.dbState.SentinelsDbState.States = []*sdlgoredis.SentinelDbState{ + &sdlgoredis.SentinelDbState{ + Fields: sdlgoredis.SentinelDbStateFields{ + Ip: ip, + Port: port, + }, + }, + } +} + func addHcMockReplicaDbState(ip, port, masterLinkOk string) { if hcMocks.dbState.ReplicasDbState == nil { hcMocks.dbState.ReplicasDbState = new(sdlgoredis.ReplicasDbState) @@ -77,6 +90,21 @@ func addHcMockReplicaDbState(ip, port, masterLinkOk string) { ) } +func addHcMockSentinelDbState(ip, port, flags string) { + if hcMocks.dbState.SentinelsDbState == nil { + hcMocks.dbState.SentinelsDbState = new(sdlgoredis.SentinelsDbState) + } + hcMocks.dbState.SentinelsDbState.States = append(hcMocks.dbState.SentinelsDbState.States, + &sdlgoredis.SentinelDbState{ + Fields: sdlgoredis.SentinelDbStateFields{ + Ip: ip, + Port: port, + Flags: flags, + }, + }, + ) +} + func newMockDatabase() *cli.Database { db := &cli.Database{} hcMocks.dbIface = new(mocks.MockDB) @@ -124,9 +152,11 @@ func TestCliHealthCheckCanShowHelp(t *testing.T) { } func TestCliHealthCheckCanShowHaDeploymentOkStatusCorrectly(t *testing.T) { - setupHcMockMasterDb("10.20.30.40", "6379", 2) + setupHcMockMasterDb("10.20.30.40", "6379") addHcMockReplicaDbState("1.2.3.4", "6379", "ok") addHcMockReplicaDbState("5.6.7.8", "6379", "ok") + addHcMockSentinelDbState("1.2.3.4", "26379", "sentinel") + addHcMockSentinelDbState("5.6.7.8", "26379", "sentinel") stdout, err := runHcCli() @@ -139,9 +169,11 @@ func TestCliHealthCheckCanShowHaDeploymentOkStatusCorrectly(t *testing.T) { } func TestCliHealthCheckCanShowHaDeploymentStatusCorrectlyWhenOneReplicaStateNotUp(t *testing.T) { - setupHcMockMasterDb("10.20.30.40", "6379", 2) + setupHcMockMasterDb("10.20.30.40", "6379") addHcMockReplicaDbState("1.2.3.4", "6379", "ok") addHcMockReplicaDbState("5.6.7.8", "6379", "nok") + addHcMockSentinelDbState("1.2.3.4", "26379", "sentinel") + addHcMockSentinelDbState("5.6.7.8", "26379", "sentinel") stdout, err := runHcCli() @@ -151,8 +183,25 @@ func TestCliHealthCheckCanShowHaDeploymentStatusCorrectlyWhenOneReplicaStateNotU assert.Contains(t, stdout, "Replica link to the master is down") } +func TestCliHealthCheckCanShowHaDeploymentStatusCorrectlyWhenOneSentinelStateNotUp(t *testing.T) { + setupHcMockMasterDb("10.20.30.40", "6379") + addHcMockReplicaDbState("1.2.3.4", "6379", "ok") + addHcMockReplicaDbState("5.6.7.8", "6379", "ok") + addHcMockSentinelDbState("1.2.3.4", "26379", "some-failure") + addHcMockSentinelDbState("5.6.7.8", "26379", "sentinel") + + stdout, err := runHcCli() + + assert.Nil(t, err) + assert.Contains(t, stdout, "Overall status: NOK") + assert.Contains(t, stdout, "Replica #1 (1.2.3.4:6379): OK") + assert.Contains(t, stdout, "Replica #2 (5.6.7.8:6379): OK") + assert.Contains(t, stdout, "Sentinel #1 (1.2.3.4:26379): NOK") + assert.Contains(t, stdout, "Sentinel flags are 'some-failure', expected 'sentinel'") +} + func TestCliHealthCheckCanShowHaDeploymentStatusCorrectlyWhenDbStateQueryFails(t *testing.T) { - setupHcMockMasterDb("10.20.30.40", "6379", 0) + setupHcMockMasterDb("10.20.30.40", "6379") hcMocks.dbErr = errors.New("Some error") expCliErr := errors.New("SDL CLI error: Some error") @@ -167,8 +216,18 @@ func TestCliHealthCheckCanShowHaDeploymentStatusCorrectlyWhenDbStateQueryFails(t assert.Contains(t, stderr, "Error: "+expCliErr.Error()) } -func TestCliHealthCheckCanShowHaDeploymentOkStatusCorrectlyWhenDbStateIsFromReplicaServer(t *testing.T) { - setupHcMockReplicaDb() +func TestCliHealthCheckCanShowHaDeploymentOkStatusCorrectlyWhenDbStateIsFromReplicaOnly(t *testing.T) { + setupHcMockReplicaDb("1.2.3.4", "6379") + + stdout, err := runHcCli() + + assert.Nil(t, err) + assert.Contains(t, stdout, "Overall status: NOK") + assert.Contains(t, stdout, "Master (): NOK") +} + +func TestCliHealthCheckCanShowHaDeploymentOkStatusCorrectlyWhenDbStateIsFromSentinelOnly(t *testing.T) { + setupHcMockSentinelDb("1.2.3.4", "26379") stdout, err := runHcCli() @@ -178,7 +237,7 @@ func TestCliHealthCheckCanShowHaDeploymentOkStatusCorrectlyWhenDbStateIsFromRepl } func TestCliHealthCheckCanShowStandaloneDeploymentOkStatusCorrectly(t *testing.T) { - setupHcMockMasterDb("10.20.30.40", "6379", 0) + setupHcMockMasterDb("10.20.30.40", "6379") stdout, err := runHcCli() diff --git a/internal/sdlgoredis/dbstate.go b/internal/sdlgoredis/dbstate.go index 543f22e..61b6fdc 100644 --- a/internal/sdlgoredis/dbstate.go +++ b/internal/sdlgoredis/dbstate.go @@ -29,8 +29,9 @@ import ( //DbState struct is a holder for DB state information, which is received from //sdlgoredis sentinel 'Master' and 'Slaves' calls output. type DbState struct { - MasterDbState MasterDbState - ReplicasDbState *ReplicasDbState + MasterDbState MasterDbState + ReplicasDbState *ReplicasDbState + SentinelsDbState *SentinelsDbState } //MasterDbState struct is a holder for master Redis state information. @@ -50,6 +51,17 @@ type ReplicaDbState struct { Fields ReplicaDbStateFields } +//SentinelsDbState struct is a holder for Redis sentinels state information. +type SentinelsDbState struct { + Err error + States []*SentinelDbState +} + +//SentinelDbState struct is a holder for one Redis sentinel state information. +type SentinelDbState struct { + Fields SentinelDbStateFields +} + //MasterDbStateFields struct is a holder for master Redis state information //fields which are read from sdlgoredis sentinel 'Master' call output. type MasterDbStateFields struct { @@ -69,6 +81,14 @@ type ReplicaDbStateFields struct { Flags string } +//SentinelDbStateFields struct is a holder for sentinel Redis state information +//fields which are read from sdlgoredis sentinel 'Sentinels' call output. +type SentinelDbStateFields struct { + Ip string + Port string + Flags string +} + func (dbst *DbState) IsOnline() error { if err := dbst.MasterDbState.IsOnline(); err != nil { return err @@ -78,6 +98,11 @@ func (dbst *DbState) IsOnline() error { return err } } + if dbst.SentinelsDbState != nil { + if err := dbst.SentinelsDbState.IsOnline(); err != nil { + return err + } + } return nil } @@ -136,3 +161,30 @@ func (rdbst *ReplicaDbState) GetAddress() string { return "" } } + +func (sdbst *SentinelsDbState) IsOnline() error { + if sdbst.Err != nil { + return sdbst.Err + } + for _, state := range sdbst.States { + if err := state.IsOnline(); err != nil { + return err + } + } + return nil +} + +func (sdbst *SentinelDbState) IsOnline() error { + if sdbst.Fields.Flags != "sentinel" { + return fmt.Errorf("Sentinel flags are '%s', expected 'sentinel'", sdbst.Fields.Flags) + } + return nil +} + +func (sdbst *SentinelDbState) GetAddress() string { + if sdbst.Fields.Ip != "" || sdbst.Fields.Port != "" { + return sdbst.Fields.Ip + ":" + sdbst.Fields.Port + } else { + return "" + } +} diff --git a/internal/sdlgoredis/dbstate_test.go b/internal/sdlgoredis/dbstate_test.go index 0b03afa..cc32309 100644 --- a/internal/sdlgoredis/dbstate_test.go +++ b/internal/sdlgoredis/dbstate_test.go @@ -68,6 +68,24 @@ func (ds *dbStateMock) addReplicaFields(role, ip, port, mls, flags string) { ds.state.ReplicasDbState.States = append(ds.state.ReplicasDbState.States, newState) } +func (ds *dbStateMock) setSentinelError(err error) { + if ds.state.SentinelsDbState == nil { + ds.state.SentinelsDbState = new(sdlgoredis.SentinelsDbState) + } + ds.state.SentinelsDbState.Err = err +} + +func (ds *dbStateMock) addSentinelFields(ip, port, flags string) { + if ds.state.SentinelsDbState == nil { + ds.state.SentinelsDbState = new(sdlgoredis.SentinelsDbState) + } + newState := new(sdlgoredis.SentinelDbState) + newState.Fields.Ip = ip + newState.Fields.Port = port + newState.Fields.Flags = flags + ds.state.SentinelsDbState.States = append(ds.state.SentinelsDbState.States, newState) +} + func TestIsOnlineWhenSingleMasterSuccessfully(t *testing.T) { st := setupDbState() st.setMasterFields("master", "1.2.3.4", "60000", "0", "master") @@ -119,6 +137,8 @@ func TestIsOnlineWhenMasterAndTwoReplicasSuccessfully(t *testing.T) { st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave") st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave") + st.addSentinelFields("6.7.8.9", "11234", "sentinel") + st.addSentinelFields("6.7.8.10", "13450", "sentinel") err := st.state.IsOnline() assert.Nil(t, err) } @@ -129,17 +149,32 @@ func TestIsOnlineWhenMasterAndTwoReplicasFailureIfErrorHasSet(t *testing.T) { st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave") st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave") + st.addSentinelFields("6.7.8.9", "11234", "sentinel") + st.addSentinelFields("6.7.8.10", "13450", "sentinel") st.setReplicaError(testErr) err := st.state.IsOnline() assert.Equal(t, testErr, err) } +func TestIsOnlineWhenMasterAndOneReplicaFailureIfSentinelErrorHasSet(t *testing.T) { + testErr := errors.New("Some error") + st := setupDbState() + st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") + st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave") + st.addSentinelFields("6.7.8.9", "11234", "sentinel") + st.setSentinelError(testErr) + err := st.state.IsOnline() + assert.Equal(t, testErr, err) +} + func TestIsOnlineWhenMasterAndTwoReplicasFailureIfNotSlaveRole(t *testing.T) { expErr := errors.New("Replica role is 'not-slave', expected 'slave'") st := setupDbState() st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave") st.addReplicaFields("not-slave", "6.7.8.10", "3450", "ok", "slave") + st.addSentinelFields("6.7.8.9", "11234", "sentinel") + st.addSentinelFields("6.7.8.10", "13450", "sentinel") err := st.state.IsOnline() assert.Equal(t, expErr, err) } @@ -150,6 +185,8 @@ func TestIsOnlineWhenMasterAndTwoReplicasFailureIfMasterLinkDown(t *testing.T) { st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") st.addReplicaFields("slave", "6.7.8.9", "1234", "nok", "slave") st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave") + st.addSentinelFields("6.7.8.9", "11234", "sentinel") + st.addSentinelFields("6.7.8.10", "13450", "sentinel") err := st.state.IsOnline() assert.Equal(t, expErr, err) } @@ -160,6 +197,18 @@ func TestIsOnlineWhenMasterAndTwoReplicasFailureIfErrorFlags(t *testing.T) { st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave") st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "any-error,slave") + st.addSentinelFields("6.7.8.9", "11234", "sentinel") + st.addSentinelFields("6.7.8.10", "13450", "sentinel") + err := st.state.IsOnline() + assert.Equal(t, expErr, err) +} + +func TestIsOnlineWhenMasterAndOneReplicaFailureIfSentinelErrorFlags(t *testing.T) { + expErr := errors.New("Sentinel flags are 'any-error,sentinel', expected 'sentinel'") + st := setupDbState() + st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") + st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave") + st.addSentinelFields("6.7.8.9", "112345", "any-error,sentinel") err := st.state.IsOnline() assert.Equal(t, expErr, err) } @@ -169,6 +218,8 @@ func TestGetAddressReplicasSuccessfully(t *testing.T) { st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave") st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave") + st.addSentinelFields("6.7.8.9", "11234", "sentinel") + st.addSentinelFields("6.7.8.10", "13450", "sentinel") addr := st.state.ReplicasDbState.States[0].GetAddress() assert.Equal(t, "6.7.8.9:1234", addr) addr = st.state.ReplicasDbState.States[1].GetAddress() @@ -180,8 +231,36 @@ func TestGetAddressReplicasNoIpPort(t *testing.T) { st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") st.addReplicaFields("slave", "", "", "ok", "slave") st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave") + st.addSentinelFields("6.7.8.9", "11234", "sentinel") + st.addSentinelFields("6.7.8.10", "13450", "sentinel") addr := st.state.ReplicasDbState.States[0].GetAddress() assert.Equal(t, "", addr) addr = st.state.ReplicasDbState.States[1].GetAddress() assert.Equal(t, "6.7.8.10:3450", addr) } + +func TestGetAddressSentinelsSuccessfully(t *testing.T) { + st := setupDbState() + st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") + st.addReplicaFields("slave", "6.7.8.9", "1234", "ok", "slave") + st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave") + st.addSentinelFields("6.7.8.9", "11234", "sentinel") + st.addSentinelFields("6.7.8.10", "13450", "sentinel") + addr := st.state.SentinelsDbState.States[0].GetAddress() + assert.Equal(t, "6.7.8.9:11234", addr) + addr = st.state.SentinelsDbState.States[1].GetAddress() + assert.Equal(t, "6.7.8.10:13450", addr) +} + +func TestGetAddressSentinelsNoIpPort(t *testing.T) { + st := setupDbState() + st.setMasterFields("master", "1.2.3.4", "60000", "2", "master") + st.addReplicaFields("slave", "", "", "ok", "slave") + st.addReplicaFields("slave", "6.7.8.10", "3450", "ok", "slave") + st.addSentinelFields("", "", "sentinel") + st.addSentinelFields("6.7.8.10", "13450", "sentinel") + addr := st.state.SentinelsDbState.States[0].GetAddress() + assert.Equal(t, "", addr) + addr = st.state.SentinelsDbState.States[1].GetAddress() + assert.Equal(t, "6.7.8.10:13450", addr) +} diff --git a/internal/sdlgoredis/sdlgoredis_test.go b/internal/sdlgoredis/sdlgoredis_test.go index 287115b..740674d 100644 --- a/internal/sdlgoredis/sdlgoredis_test.go +++ b/internal/sdlgoredis/sdlgoredis_test.go @@ -149,11 +149,17 @@ func (m *MockRedisSentinel) Master(name string) *redis.StringStringMapCmd { a := m.Called(name) return a.Get(0).(*redis.StringStringMapCmd) } + func (m *MockRedisSentinel) Slaves(name string) *redis.SliceCmd { a := m.Called(name) return a.Get(0).(*redis.SliceCmd) } +func (m *MockRedisSentinel) Sentinels(name string) *redis.SliceCmd { + a := m.Called(name) + return a.Get(0).(*redis.SliceCmd) +} + func setSubscribeNotifications() (*pubSubMock, sdlgoredis.SubscribeFn) { mock := new(pubSubMock) return mock, func(client sdlgoredis.RedisClient, channels ...string) sdlgoredis.Subscriber { @@ -1138,6 +1144,17 @@ func TestStateWithMasterAndTwoSlaveRedisSuccessfully(t *testing.T) { "port", "30000", "role-reported", "slave", } + redisSentinelsState := make([]interface{}, 2) + redisSentinelsState[0] = []interface{}{ + "ip", "10.20.30.40", + "port", "26379", + "flags", "sentinel", + } + redisSentinelsState[1] = []interface{}{ + "ip", "10.20.30.50", + "flags", "sentinel", + "port", "30001", + } expState := &sdlgoredis.DbState{ MasterDbState: sdlgoredis.MasterDbState{ @@ -1167,10 +1184,29 @@ func TestStateWithMasterAndTwoSlaveRedisSuccessfully(t *testing.T) { }, }, }, + SentinelsDbState: &sdlgoredis.SentinelsDbState{ + States: []*sdlgoredis.SentinelDbState{ + &sdlgoredis.SentinelDbState{ + Fields: sdlgoredis.SentinelDbStateFields{ + Ip: "10.20.30.40", + Port: "26379", + Flags: "sentinel", + }, + }, + &sdlgoredis.SentinelDbState{ + Fields: sdlgoredis.SentinelDbStateFields{ + Ip: "10.20.30.50", + Port: "30001", + Flags: "sentinel", + }, + }, + }, + }, } s[0].On("Master", "dbaasmaster").Return(redis.NewStringStringMapResult(redisMasterState, nil)) s[0].On("Slaves", "dbaasmaster").Return(redis.NewSliceResult(redisSlavesState, nil)) + s[0].On("Sentinels", "dbaasmaster").Return(redis.NewSliceResult(redisSentinelsState, nil)) ret, err := db.State() assert.Nil(t, err) assert.Equal(t, expState, ret) @@ -1188,6 +1224,12 @@ func TestStateWithMasterAndOneSlaveRedisFailureInMasterRedisCall(t *testing.T) { "flags", "slave", "master-link-status", "up", } + redisSentinelsState := make([]interface{}, 1) + redisSentinelsState[0] = []interface{}{ + "ip", "10.20.30.40", + "port", "26379", + "flags", "sentinel", + } expState := &sdlgoredis.DbState{ MasterDbState: sdlgoredis.MasterDbState{ @@ -1206,10 +1248,22 @@ func TestStateWithMasterAndOneSlaveRedisFailureInMasterRedisCall(t *testing.T) { }, }, }, + SentinelsDbState: &sdlgoredis.SentinelsDbState{ + States: []*sdlgoredis.SentinelDbState{ + &sdlgoredis.SentinelDbState{ + Fields: sdlgoredis.SentinelDbStateFields{ + Ip: "10.20.30.40", + Port: "26379", + Flags: "sentinel", + }, + }, + }, + }, } s[0].On("Master", "dbaasmaster").Return(redis.NewStringStringMapResult(redisMasterState, errors.New("Some error"))) s[0].On("Slaves", "dbaasmaster").Return(redis.NewSliceResult(redisSlavesState, nil)) + s[0].On("Sentinels", "dbaasmaster").Return(redis.NewSliceResult(redisSentinelsState, nil)) ret, err := db.State() assert.NotNil(t, err) assert.Equal(t, expState, ret) @@ -1223,6 +1277,12 @@ func TestStateWithMasterAndOneSlaveRedisFailureInSlavesRedisCall(t *testing.T) { } redisSlavesState := make([]interface{}, 1) redisSlavesState[0] = []interface{}{} + redisSentinelsState := make([]interface{}, 1) + redisSentinelsState[0] = []interface{}{ + "ip", "10.20.30.40", + "port", "26379", + "flags", "sentinel", + } expState := &sdlgoredis.DbState{ MasterDbState: sdlgoredis.MasterDbState{ @@ -1234,10 +1294,76 @@ func TestStateWithMasterAndOneSlaveRedisFailureInSlavesRedisCall(t *testing.T) { Err: errors.New("Some error"), States: []*sdlgoredis.ReplicaDbState{}, }, + SentinelsDbState: &sdlgoredis.SentinelsDbState{ + States: []*sdlgoredis.SentinelDbState{ + &sdlgoredis.SentinelDbState{ + Fields: sdlgoredis.SentinelDbStateFields{ + Ip: "10.20.30.40", + Port: "26379", + Flags: "sentinel", + }, + }, + }, + }, } s[0].On("Master", "dbaasmaster").Return(redis.NewStringStringMapResult(redisMasterState, nil)) s[0].On("Slaves", "dbaasmaster").Return(redis.NewSliceResult(redisSlavesState, errors.New("Some error"))) + s[0].On("Sentinels", "dbaasmaster").Return(redis.NewSliceResult(redisSentinelsState, nil)) + ret, err := db.State() + assert.NotNil(t, err) + assert.Equal(t, expState, ret) + r.AssertExpectations(t) +} + +func TestStateWithMasterAndOneSlaveRedisFailureInSentinelsRedisCall(t *testing.T) { + _, r, s, db := setupHaEnvWithSentinels(true) + redisMasterState := map[string]string{ + "role-reported": "master", + } + redisSlavesState := make([]interface{}, 1) + redisSlavesState[0] = []interface{}{ + "role-reported", "slave", + "ip", "10.20.30.40", + "port", "6379", + "flags", "slave", + "master-link-status", "up", + } + redisSentinelsState := make([]interface{}, 1) + redisSentinelsState[0] = []interface{}{ + "ip", "10.20.30.40", + "port", "26379", + "flags", "sentinel", + } + + expState := &sdlgoredis.DbState{ + MasterDbState: sdlgoredis.MasterDbState{ + Fields: sdlgoredis.MasterDbStateFields{ + Role: "master", + }, + }, + ReplicasDbState: &sdlgoredis.ReplicasDbState{ + States: []*sdlgoredis.ReplicaDbState{ + &sdlgoredis.ReplicaDbState{ + Fields: sdlgoredis.ReplicaDbStateFields{ + Role: "slave", + Ip: "10.20.30.40", + Port: "6379", + MasterLinkStatus: "up", + Flags: "slave", + }, + }, + }, + }, + SentinelsDbState: &sdlgoredis.SentinelsDbState{ + Err: errors.New("Some error"), + States: []*sdlgoredis.SentinelDbState{}, + }, + } + + s[0].On("Master", "dbaasmaster").Return(redis.NewStringStringMapResult(redisMasterState, nil)) + s[0].On("Slaves", "dbaasmaster").Return(redis.NewSliceResult(redisSlavesState, nil)) + s[0].On("Sentinels", "dbaasmaster").Return(redis.NewSliceResult(redisSentinelsState, errors.New("Some error"))) ret, err := db.State() assert.NotNil(t, err) assert.Equal(t, expState, ret) diff --git a/internal/sdlgoredis/sdlgosentinel.go b/internal/sdlgoredis/sdlgosentinel.go index 7af0410..014d4ba 100644 --- a/internal/sdlgoredis/sdlgosentinel.go +++ b/internal/sdlgoredis/sdlgosentinel.go @@ -34,6 +34,7 @@ type Sentinel struct { type IredisSentinelClient interface { Master(name string) *redis.StringStringMapCmd Slaves(name string) *redis.SliceCmd + Sentinels(name string) *redis.SliceCmd } type RedisSentinelCreateCb func(cfg *Config, addr string) *Sentinel @@ -56,12 +57,17 @@ func (s *Sentinel) GetDbState() (*DbState, error) { state := new(DbState) mState, mErr := s.getMasterDbState() rState, rErr := s.getReplicasState() + sState, sErr := s.getSentinelsState() state.MasterDbState = *mState state.ReplicasDbState = rState - if mErr == nil { + state.SentinelsDbState = sState + if mErr != nil { + return state, mErr + } + if rErr != nil { return state, rErr } - return state, mErr + return state, sErr } func (s *Sentinel) getMasterDbState() (*MasterDbState, error) { @@ -109,3 +115,32 @@ func readReplicaState(redisSlaves []interface{}) *ReplicaDbState { } return state } + +func (s *Sentinel) getSentinelsState() (*SentinelsDbState, error) { + states := new(SentinelsDbState) + states.States = make([]*SentinelDbState, 0) + + redisVal, redisErr := s.Sentinels(s.Cfg.masterName).Result() + if redisErr == nil { + for _, redisSentinel := range redisVal { + sentinelState := readSentinelState(redisSentinel.([]interface{})) + states.States = append(states.States, sentinelState) + } + } + states.Err = redisErr + return states, redisErr +} + +func readSentinelState(redisSentinels []interface{}) *SentinelDbState { + state := new(SentinelDbState) + for i := 0; i < len(redisSentinels); i += 2 { + if redisSentinels[i].(string) == "ip" { + state.Fields.Ip = redisSentinels[i+1].(string) + } else if redisSentinels[i].(string) == "port" { + state.Fields.Port = redisSentinels[i+1].(string) + } else if redisSentinels[i].(string) == "flags" { + state.Fields.Flags = redisSentinels[i+1].(string) + } + } + return state +} -- 2.16.6