X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=manager%2Fcmd%2Fmanager_test.go;h=d4ab43753afa76ee4aeb16b8a00bb24c833bda95;hb=bd09e69f279ab1566cd92a8d6d7b5ab58418b18f;hp=838de7914444f5979f8600012a8169cda1a5b1d9;hpb=4cedd5020e5c7ba0383156f9929ab3db986c0593;p=ric-plt%2Falarm-go.git diff --git a/manager/cmd/manager_test.go b/manager/cmd/manager_test.go index 838de79..d4ab437 100755 --- a/manager/cmd/manager_test.go +++ b/manager/cmd/manager_test.go @@ -24,21 +24,23 @@ import ( "bytes" "encoding/json" "fmt" - "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm" - "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp" - "github.com/gorilla/mux" - "github.com/prometheus/alertmanager/api/v2/models" - "github.com/stretchr/testify/assert" "io" "io/ioutil" "net" "net/http" "net/http/httptest" "os" + "os/exec" "strconv" "strings" "testing" "time" + + "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm" + "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp" + "github.com/gorilla/mux" + "github.com/prometheus/alertmanager/api/v2/models" + "github.com/stretchr/testify/assert" ) var alarmManager *AlarmManager @@ -47,10 +49,10 @@ var eventChan chan string // Test cases func TestMain(M *testing.M) { - os.Setenv("ALARM_IF_RMR", "true") - alarmManager = NewAlarmManager("localhost:9093", 500) - go alarmManager.Run(false) - time.Sleep(time.Duration(2) * time.Second) + alarmManager = NewAlarmManager("localhost:9093", 500, false) + alarmManager.alertInterval = 20000 + go alarmManager.Run(false, 5) + time.Sleep(time.Duration(10) * time.Second) // Wait until RMR is up-and-running for !xapp.Rmr.IsReady() { @@ -65,66 +67,120 @@ func TestMain(M *testing.M) { os.Exit(M.Run()) } +func TestGetPreDefinedAlarmDefinitions(t *testing.T) { + xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions") + var alarmDefinition alarm.AlarmDefinition + req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil) + vars := map[string]string{"alarmId": strconv.FormatUint(72004, 10)} + req = mux.SetURLVars(req, vars) + handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition) + response := executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + json.NewDecoder(response.Body).Decode(&alarmDefinition) + xapp.Logger.Info("alarm definition = %v", alarmDefinition) + if alarmDefinition.AlarmId != alarm.E2_CONNECTION_PROBLEM || alarmDefinition.AlarmText != "E2 CONNECTION PROBLEM" { + t.Errorf("Incorrect alarm definition") + } +} + func TestSetAlarmDefinitions(t *testing.T) { xapp.Logger.Info("TestSetAlarmDefinitions") - var alarm8004Definition alarm.AlarmDefinition - alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED - alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED" - alarm8004Definition.EventType = "Processing error" - alarm8004Definition.OperationInstructions = "Not defined" - - var alarm8005Definition alarm.AlarmDefinition - alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS - alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS" - alarm8005Definition.EventType = "Communication error" - alarm8005Definition.OperationInstructions = "Not defined" - - var alarm8006Definition alarm.AlarmDefinition - alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB - alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB" - alarm8006Definition.EventType = "Communication error" - alarm8006Definition.OperationInstructions = "Not defined" - - var alarm8007Definition alarm.AlarmDefinition - alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB - alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB" - alarm8007Definition.EventType = "Communication error" - alarm8007Definition.OperationInstructions = "Not defined" - - var alarm8008Definition alarm.AlarmDefinition - alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD - alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD" - alarm8008Definition.EventType = "storage warning" - alarm8008Definition.OperationInstructions = "clear alarms or raise threshold" - - var alarm8009Definition alarm.AlarmDefinition - alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD - alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD" - alarm8009Definition.EventType = "storage warning" - alarm8009Definition.OperationInstructions = "clear alarms or raise threshold" - - pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}} + + var alarm72004Definition alarm.AlarmDefinition + alarm72004Definition.AlarmId = alarm.E2_CONNECTION_PROBLEM + alarm72004Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB/G-NODEB" + alarm72004Definition.EventType = "Communication error" + alarm72004Definition.OperationInstructions = "Not defined" + alarm72004Definition.RaiseDelay = 0 + alarm72004Definition.ClearDelay = 0 + + var alarm72008Definition alarm.AlarmDefinition + alarm72008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD + alarm72008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD" + alarm72008Definition.EventType = "storage warning" + alarm72008Definition.OperationInstructions = "Clear alarms or raise threshold" + alarm72008Definition.RaiseDelay = 0 + alarm72008Definition.ClearDelay = 0 + + var alarm72009Definition alarm.AlarmDefinition + alarm72009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD + alarm72009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD" + alarm72009Definition.EventType = "storage warning" + alarm72009Definition.OperationInstructions = "Clear alarms or raise threshold" + alarm72009Definition.RaiseDelay = 0 + alarm72009Definition.ClearDelay = 0 + + pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm72004Definition, &alarm72008Definition, &alarm72009Definition}} pbodyEn, _ := json.Marshal(pbodyParams) req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn)) handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition) response := executeRequest(req, handleFunc) status := checkResponseCode(t, http.StatusOK, response.Code) xapp.Logger.Info("status = %v", status) +} + +func TestSetAlarmConfigDecodeError(t *testing.T) { + xapp.Logger.Info("TestSetAlarmConfigDecodeError") + + var jsonStr = []byte(`{"test":"Invalid Alarm Data", "test1" 123}`) + req, _ := http.NewRequest("POST", "/ric/v1/alarms", bytes.NewBuffer(jsonStr)) + handleFunc := http.HandlerFunc(alarmManager.SetAlarmConfig) + response := executeRequest(req, handleFunc) + status := checkResponseCode(t,http.StatusOK,response.Code) + xapp.Logger.Info("status = %v", status) +} + +func TestSetAlarmDefinitionDecodeError(t *testing.T) { + xapp.Logger.Info("TestSetAlarmDefinitionDecodeError") + + var jsonStr = []byte(`{"test":"Invalid Alarm Data", "test1" 123}`) + req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(jsonStr)) + handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition) + response := executeRequest(req, handleFunc) + status := checkResponseCode(t,http.StatusBadRequest,response.Code) + xapp.Logger.Info("status = %v", status) +} + +func TestRaiseAlarmEmptyBody(t *testing.T) { + xapp.Logger.Info("TestRaiseAlarmEmptyBody") + req, _ := http.NewRequest("POST", "/ric/v1/alarms", nil) + handleFunc := http.HandlerFunc(alarmManager.RaiseAlarm) + response := executeRequest(req, handleFunc) + status := checkResponseCode(t,http.StatusOK,response.Code) + xapp.Logger.Info("status = %v", status) +} + +func TestSetAlarmDefinitionsEmptyBody(t *testing.T) { + xapp.Logger.Info("TestSetAlarmDefinitionsEmptyBody") + req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", nil) + handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition) + response := executeRequest(req, handleFunc) + status := checkResponseCode(t,http.StatusBadRequest,response.Code) + xapp.Logger.Info("status = %v", status) +} + +func TestClearAlarmEmptyBody(t *testing.T) { + xapp.Logger.Info("TestClearAlarmEmptyBody") + req, _ := http.NewRequest("DELETE", "/ric/v1/alarms", nil) + handleFunc := http.HandlerFunc(alarmManager.ClearAlarm) + response := executeRequest(req, handleFunc) + status := checkResponseCode(t,http.StatusOK,response.Code) + xapp.Logger.Info("status = %v", status) } func TestGetAlarmDefinitions(t *testing.T) { xapp.Logger.Info("TestGetAlarmDefinitions") var alarmDefinition alarm.AlarmDefinition req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil) - vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)} + vars := map[string]string{"alarmId": strconv.FormatUint(72004, 10)} req = mux.SetURLVars(req, vars) handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition) response := executeRequest(req, handleFunc) checkResponseCode(t, http.StatusOK, response.Code) json.NewDecoder(response.Body).Decode(&alarmDefinition) xapp.Logger.Info("alarm definition = %v", alarmDefinition) - if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" { + if alarmDefinition.AlarmId != alarm.E2_CONNECTION_PROBLEM || alarmDefinition.AlarmText != "E2 CONNECTION PROBLEM" { t.Errorf("Incorrect alarm definition") } } @@ -143,38 +199,56 @@ func TestDeleteAlarmDefinitions(t *testing.T) { xapp.Logger.Info("alarm definition = %v", *alarmDefinition) } - //Delete 8004 + //Delete 72004 req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil) - vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)} + vars := map[string]string{"alarmId": strconv.FormatUint(72004, 10)} req = mux.SetURLVars(req, vars) handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition) response = executeRequest(req, handleFunc) checkResponseCode(t, http.StatusOK, response.Code) - //Get 8004 fail + //Get 72004 fail req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil) - vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)} + vars = map[string]string{"alarmId": strconv.FormatUint(72004, 10)} req = mux.SetURLVars(req, vars) handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition) response = executeRequest(req, handleFunc) checkResponseCode(t, http.StatusBadRequest, response.Code) - //Set 8004 success - var alarm8004Definition alarm.AlarmDefinition - alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED - alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED" - alarm8004Definition.EventType = "Processing error" - alarm8004Definition.OperationInstructions = "Not defined" - pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}} + //Delete Alarm which doesn't present + req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil) + vars = map[string]string{"alarmId": strconv.FormatUint(882004, 10)} + req = mux.SetURLVars(req, vars) + handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition) + response = executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + //Delete Alarm which is incorrect present + req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil) + vars = map[string]string{"alarmId": strconv.FormatUint(1234, 8)} + req = mux.SetURLVars(req, vars) + handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition) + response = executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + //Set 72004 success + var alarm72004Definition alarm.AlarmDefinition + alarm72004Definition.AlarmId = alarm.E2_CONNECTION_PROBLEM + alarm72004Definition.AlarmText = "E2 CONNECTION PROBLEM" + alarm72004Definition.EventType = "Processing error" + alarm72004Definition.OperationInstructions = "Not defined" + alarm72004Definition.RaiseDelay = 0 + alarm72004Definition.ClearDelay = 0 + pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm72004Definition}} pbodyEn, _ := json.Marshal(pbodyParams) req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn)) handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition) response = executeRequest(req, handleFunc) checkResponseCode(t, http.StatusOK, response.Code) - //Get 8004 success + //Get 72004 success req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil) - vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)} + vars = map[string]string{"alarmId": strconv.FormatUint(72004, 10)} req = mux.SetURLVars(req, vars) handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition) response = executeRequest(req, handleFunc) @@ -186,10 +260,38 @@ func TestNewAlarmStoredAndPostedSucess(t *testing.T) { ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{}) defer ts.Close() - a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1") + a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityCritical, "Some App data", "eth 0 1") assert.Nil(t, alarmer.Raise(a), "raise failed") VerifyAlarm(t, a, 1) + + var activeAlarms []AlarmNotification + activeAlarms = make([]AlarmNotification, 1) + req, _ := http.NewRequest("GET", "/ric/v1/alarms/active", nil) + req = mux.SetURLVars(req, nil) + handleFunc := http.HandlerFunc(alarmManager.GetActiveAlarms) + response := executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + // Decode the json output from handler + json.NewDecoder(response.Body).Decode(activeAlarms) + if len(activeAlarms) != 1 { + t.Errorf("Incorrect alarm alarm count") + } + + var alarmHistory []AlarmNotification + alarmHistory = make([]AlarmNotification, 1) + req, _ = http.NewRequest("GET", "/ric/v1/alarms/history", nil) + req = mux.SetURLVars(req, nil) + handleFunc = http.HandlerFunc(alarmManager.GetAlarmHistory) + response = executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + // Decode the json output from handler + json.NewDecoder(response.Body).Decode(alarmHistory) + if len(alarmHistory) != 1 { + t.Errorf("Incorrect alarm history count") + } } func TestAlarmClearedSucess(t *testing.T) { @@ -198,14 +300,14 @@ func TestAlarmClearedSucess(t *testing.T) { defer ts.Close() // Raise the alarm - a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1") + a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityCritical, "Some App data", "eth 0 1") assert.Nil(t, alarmer.Raise(a), "raise failed") VerifyAlarm(t, a, 1) // Now Clear the alarm and check alarm is removed - a = alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityCritical, "Some App data", "eth 0 1") - assert.Nil(t, alarmer.Clear(a), "clear failed") + a = alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityCritical, "Some App data", "eth 0 1") + assert.Nil(t, alarmer.Clear(a), "clear failed") time.Sleep(time.Duration(2) * time.Second) assert.Equal(t, len(alarmManager.activeAlarms), 0) @@ -217,13 +319,17 @@ func TestMultipleAlarmsRaisedSucess(t *testing.T) { defer ts.Close() // Raise two alarms - a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1") - assert.Nil(t, alarmer.Raise(a), "raise failed") + a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1") + assert.Nil(t, alarmer.Raise(a), "raise failed") - b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11") + b := alarmer.NewAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, alarm.SeverityMinor, "Hello", "abcd 11") assert.Nil(t, alarmer.Raise(b), "raise failed") - VerifyAlarm(t, a, 2) + time.Sleep(time.Duration(5) * time.Second) + + xapp.Logger.Info("VerifyAlarm: %d %+v", len(alarmManager.activeAlarms), alarmManager.activeAlarms) + VerifyAlarm(t, a, 1) + xapp.Logger.Info("VerifyAlarm: %d %+v", len(alarmManager.activeAlarms), alarmManager.activeAlarms) VerifyAlarm(t, b, 2) } @@ -233,10 +339,10 @@ func TestMultipleAlarmsClearedSucess(t *testing.T) { defer ts.Close() // Raise two alarms - a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1") + a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1") assert.Nil(t, alarmer.Clear(a), "clear failed") - b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11") + b := alarmer.NewAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, alarm.SeverityMinor, "Hello", "abcd 11") assert.Nil(t, alarmer.Clear(b), "clear failed") time.Sleep(time.Duration(2) * time.Second) @@ -249,7 +355,7 @@ func TestAlarmsSuppresedSucess(t *testing.T) { defer ts.Close() // Raise two similar/matching alarms ... the second one suppresed - a := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1") + a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1") assert.Nil(t, alarmer.Raise(a), "raise failed") assert.Nil(t, alarmer.Raise(a), "raise failed") @@ -288,7 +394,7 @@ func TestActiveAlarmMaxThresholds(t *testing.T) { alarmManager.maxActiveAlarms = 0 alarmManager.maxAlarmHistory = 10 - a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2") + a := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityCritical, "Some Application data", "eth 0 2") assert.Nil(t, alarmer.Raise(a), "raise failed") var alarmConfigParams alarm.AlarmConfigParams @@ -314,10 +420,295 @@ func TestActiveAlarmMaxThresholds(t *testing.T) { ts.Close() } +func TestGetPrometheusAlerts(t *testing.T) { + time.Sleep(1 * time.Second) + xapp.Logger.Info("TestGetPrometheusAlerts") + ts := CreatePromAlertSimulator2(t, "GET", "/alerts?active=true&inhibited=true&silenced=true&unprocessed=true") + + commandReady := make(chan bool, 1) + command := "cli/alarm-cli" + args := []string{"alerts", "--active", "true", "--inhibited", "true", "--silenced", "--unprocessed", "true", "true", "--host", "localhost", "--port", "9093", "flushall"} + ExecCLICommand(commandReady, command, args...) + <-commandReady + + ts.Close() +} + +func TestDelayedAlarmRaiseAndClear(t *testing.T) { + xapp.Logger.Info("TestDelayedAlarmRaiseAndClear") + + activeAlarmsBeforeTest := len(alarmManager.activeAlarms) + alarmHistoryBeforeTest := len(alarmManager.alarmHistory) + + // Add new alarm definition + var alarm9999Definition alarm.AlarmDefinition + alarm9999Definition.AlarmId = 9999 + alarm9999Definition.AlarmText = "DELAYED TEST ALARM" + alarm9999Definition.EventType = "Test type" + alarm9999Definition.OperationInstructions = "Not defined" + alarm9999Definition.RaiseDelay = 1 + alarm9999Definition.ClearDelay = 1 + pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}} + pbodyEn, _ := json.Marshal(pbodyParams) + req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn)) + handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition) + response := executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + // Verify 9999 alarm definition + req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil) + vars := map[string]string{"alarmId": strconv.FormatUint(72004, 10)} + req = mux.SetURLVars(req, vars) + handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition) + response = executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{}) + defer ts.Close() + + // Raise alarm. Posting alert and updating alarm history should be delayed + a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1") + assert.Nil(t, alarmer.Raise(a), "raise failed") + VerifyAlarm(t, a, activeAlarmsBeforeTest+1) + + // Clear the alarm and check the alarm is removed. Posting alert clear and updating alarm history should be delayed + assert.Nil(t, alarmer.Clear(a), "clear failed") + + time.Sleep(time.Duration(2) * time.Second) + assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest) + assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+2) +} + +func TestDelayedAlarmRaiseAndClear2(t *testing.T) { + xapp.Logger.Info("TestDelayedAlarmRaiseAndClear2") + + activeAlarmsBeforeTest := len(alarmManager.activeAlarms) + alarmHistoryBeforeTest := len(alarmManager.alarmHistory) + + ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{}) + defer ts.Close() + + // Raise two alarms. The first should be delayed + a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1") + assert.Nil(t, alarmer.Raise(a), "raise failed") + VerifyAlarm(t, a, activeAlarmsBeforeTest+1) + + b := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1") + assert.Nil(t, alarmer.Raise(b), "raise failed") + VerifyAlarm(t, b, activeAlarmsBeforeTest+2) + + // Clear two alarms. The first should be delayed. Check the alarms are removed + assert.Nil(t, alarmer.Clear(a), "clear failed") + assert.Nil(t, alarmer.Clear(b), "clear failed") + + time.Sleep(time.Duration(2) * time.Second) + assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest) + assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4) +} + +func TestDelayedAlarmRaiseAndClear3(t *testing.T) { + xapp.Logger.Info("TestDelayedAlarmRaiseAndClear3") + + // Delete exisitng 9999 alarm definition + req, _ := http.NewRequest("DELETE", "/ric/v1/alarms/define", nil) + vars := map[string]string{"alarmId": strconv.FormatUint(9999, 10)} + req = mux.SetURLVars(req, vars) + handleFunc := http.HandlerFunc(alarmManager.DeleteAlarmDefinition) + response := executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + // Add updated 9999 alarm definition + var alarm9999Definition alarm.AlarmDefinition + alarm9999Definition.AlarmId = 9999 + alarm9999Definition.AlarmText = "DELAYED TEST ALARM" + alarm9999Definition.EventType = "Test type" + alarm9999Definition.OperationInstructions = "Not defined" + alarm9999Definition.RaiseDelay = 1 + alarm9999Definition.ClearDelay = 0 + pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}} + pbodyEn, _ := json.Marshal(pbodyParams) + req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn)) + handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition) + response = executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + // Verify 9999 alarm definition + req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil) + vars = map[string]string{"alarmId": strconv.FormatUint(72004, 10)} + req = mux.SetURLVars(req, vars) + handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition) + response = executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + activeAlarmsBeforeTest := len(alarmManager.activeAlarms) + alarmHistoryBeforeTest := len(alarmManager.alarmHistory) + + ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{}) + defer ts.Close() + + // Raise two alarms. The first should be delayed + a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1") + assert.Nil(t, alarmer.Raise(a), "raise failed") + VerifyAlarm(t, a, activeAlarmsBeforeTest+1) + + b := alarmer.NewAlarm(alarm.E2_CONNECTION_PROBLEM, alarm.SeverityMajor, "Some App data", "eth 0 1") + assert.Nil(t, alarmer.Raise(b), "raise failed") + VerifyAlarm(t, b, activeAlarmsBeforeTest+2) + + // Clear two alarms. The first should be delayed. Check the alarms are removed + assert.Nil(t, alarmer.Clear(a), "clear failed") + assert.Nil(t, alarmer.Clear(b), "clear failed") + + time.Sleep(time.Duration(2) * time.Second) + assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest) + assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4) +} + +func TestClearExpiredAlarms(t *testing.T) { + xapp.Logger.Info("TestClearExpiredAlarms") + + a := alarm.AlarmMessage{ + Alarm: alarmer.NewAlarm(72004, alarm.SeverityWarning, "threshold", ""), + AlarmAction: alarm.AlarmActionRaise, + AlarmTime: time.Now().UnixNano(), + } + d := alarm.RICAlarmDefinitions[72004] + n := AlarmNotification{a, *d} + alarmManager.activeAlarms = make([]AlarmNotification, 0) + alarmManager.UpdateActiveAlarmList(&n) + + // Unknown SP + a.Alarm.SpecificProblem = 1234 + assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed") + + // TTL is 0 + d.TimeToLive = 0 + assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed") + + // TTL not expired + a.Alarm.SpecificProblem = 72004 + d.TimeToLive = 2 + assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed") + + // TTL expired, alarm should be cleared + time.Sleep(time.Duration(3) * time.Second) + assert.Equal(t, len(alarmManager.activeAlarms), 1) + assert.True(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed") + assert.Equal(t, len(alarmManager.activeAlarms), 0) +} + +func TestSetAlarmConfig(t *testing.T) { + xapp.Logger.Info("TestSetAlarmConfig") + + var setAlarmConfig alarm.AlarmConfigParams + setAlarmConfig.MaxActiveAlarms = 500 + setAlarmConfig.MaxAlarmHistory = 2000 + + pbodyEn, _ := json.Marshal(setAlarmConfig) + req, _ := http.NewRequest("POST", "/ric/v1/alarms/config", bytes.NewBuffer(pbodyEn)) + handleFunc := http.HandlerFunc(alarmManager.SetAlarmConfig) + response := executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + var getAlarmConfig alarm.AlarmConfigParams + req, _ = http.NewRequest("GET", "/ric/v1/alarms/config", nil) + req = mux.SetURLVars(req, nil) + handleFunc = http.HandlerFunc(alarmManager.GetAlarmConfig) + response = executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + // Decode the json output from handler + json.NewDecoder(response.Body).Decode(&getAlarmConfig) + if getAlarmConfig.MaxActiveAlarms != 500 || getAlarmConfig.MaxAlarmHistory != 2000 { + t.Errorf("Incorrect alarm thresholds") + } + + // Revert ot default + setAlarmConfig.MaxActiveAlarms = 5000 + setAlarmConfig.MaxAlarmHistory = 20000 + + pbodyEn, _ = json.Marshal(setAlarmConfig) + req, _ = http.NewRequest("POST", "/ric/v1/alarms/config", bytes.NewBuffer(pbodyEn)) + handleFunc = http.HandlerFunc(alarmManager.SetAlarmConfig) + response = executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + req, _ = http.NewRequest("GET", "/ric/v1/alarms/config", nil) + req = mux.SetURLVars(req, nil) + handleFunc = http.HandlerFunc(alarmManager.GetAlarmConfig) + response = executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + // Decode the json output from handler + json.NewDecoder(response.Body).Decode(&getAlarmConfig) + if getAlarmConfig.MaxActiveAlarms != 5000 || getAlarmConfig.MaxAlarmHistory != 20000 { + t.Errorf("Incorrect alarm thresholds") + } +} + +func TestConfigChangeCB(t *testing.T) { + xapp.Logger.Info("TestConfigChangeCB") + alarmManager.ConfigChangeCB("AlarmManager") +} + +func TestPersistentStorage(t *testing.T) { + xapp.Logger.Info("TestPersistentStorage") + alarmManager.alarmInfoPvFile = "../../definitions/sample.json" + alarmManager.ReadAlarmInfoFromPersistentVolume() +} + +func TestReadAlarmDefinitionFromJsonWrongFilename(t *testing.T) { + // use to set wrong file name os.Setenv("SITE_TITLE", "Test Site") + xapp.Logger.Info("TestReadAlarmDefinitionFromJsonWrongFilename") + os.Setenv("DEF_FILE","test.json") + alarmManager.ReadAlarmDefinitionFromJson() + // correct the filename +} + +func TestReadAlarmDefinitionFromJsonInvalidFilename(t *testing.T) { + // use to set wrong file name os.Setenv("SITE_TITLE", "Test Site") + xapp.Logger.Info("TestReadAlarmDefinitionFromJsonInvalidFilename") + os.Setenv("DEF_FILE","../../definitions/test.json") + alarmManager.ReadAlarmDefinitionFromJson() + // correct the filename +} + +func TestPostAlarm(t *testing.T) { + xapp.Logger.Info("TestPostAlarm") + var activeAlarms []AlarmNotification + activeAlarms = make([]AlarmNotification, 1) + alarmManager.PostAlarm(&activeAlarms[0]) +} + +func TestPostAlarm1(t *testing.T) { + xapp.Logger.Info("TestPostAlarm") + var activeAlarms []AlarmNotification + activeAlarms = make([]AlarmNotification, 2) + alarmManager.PostAlarm(&activeAlarms[0]) +} + +func TestNewAlarmManagerOther(t *testing.T){ + NewAlarmManager("", 0, true) +} + +func TestStatusCallbackFailure(t *testing.T) { + xapp.Logger.Info("TestStatusCallbackFailure") + alarmManager.rmrReady = false + assert.Equal(t, false, alarmManager.StatusCB()) +} + +func TestConfigChangeCBFailure(t *testing.T) { + xapp.Logger.Info("TestConfigChangeCBFailure") + alarmManager.maxActiveAlarms = 0 + alarmManager.maxAlarmHistory = 0 + alarmManager.ConfigChangeCB("AlarmManager") +} + + func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string { receivedAlert := waitForEvent() - assert.Equal(t, len(alarmManager.activeAlarms), expectedCount) + assert.Equal(t, expectedCount, len(alarmManager.activeAlarms)) _, ok := alarmManager.IsMatchFound(a) assert.True(t, ok) @@ -353,6 +744,33 @@ func CreatePromAlertSimulator(t *testing.T, method, url string, status int, resp return ts } +func CreatePromAlertSimulator2(t *testing.T, method, url string) *httptest.Server { + l, err := net.Listen("tcp", "localhost:9093") + if err != nil { + t.Error("Failed to create listener: " + err.Error()) + } + ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.Method, method) + assert.Equal(t, r.URL.String(), url) + + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(200) + // Read alerts from file + payload, err := readJSONFromFile("../testresources/prometheus-alerts.json") + if err != nil { + t.Error("Failed to send response: ", err) + } + _, err = w.Write(payload) + if err != nil { + t.Error("Failed to send response: " + err.Error()) + } + })) + ts.Listener.Close() + ts.Listener = l + ts.Start() + return ts +} + func waitForEvent() string { receivedAlert := <-eventChan return receivedAlert @@ -381,3 +799,29 @@ func checkResponseCode(t *testing.T, expected, actual int) bool { } return true } + +func ExecCLICommand(commandReady chan bool, command string, args ...string) { + go func() { + xapp.Logger.Info("Giving CLI command") + cmd := exec.Command(command, args...) + cmd.Dir = "../" + output, err := cmd.CombinedOutput() + if err != nil { + xapp.Logger.Info("CLI command failed out: %s", err) + } + xapp.Logger.Info("CLI command output: %s", output) + commandReady <- true + xapp.Logger.Info("CLI command completed") + }() +} + +func readJSONFromFile(filename string) ([]byte, error) { + file, err := ioutil.ReadFile(filename) + if err != nil { + err := fmt.Errorf("readJSONFromFile() failed: Error: %v", err) + return nil, err + } + return file, nil +} + +