X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=blobdiff_plain;f=manager%2Fcmd%2Fmanager_test.go;h=0457c58fdb4a940086b9636d871973a035e4ee03;hb=43fe17ef30078c8ceb624a4aa5167d509466e086;hp=883b112eb3928dfa4d84c14485da8cd6bcd2d82e;hpb=663b8e1a9cb5add7ea8bf6ec33696243e7d7d0dc;p=ric-plt%2Falarm-go.git

diff --git a/manager/cmd/manager_test.go b/manager/cmd/manager_test.go
index 883b112..0457c58 100755
--- a/manager/cmd/manager_test.go
+++ b/manager/cmd/manager_test.go
@@ -35,6 +35,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"os/exec"
 	"strconv"
 	"strings"
 	"testing"
@@ -47,9 +48,9 @@ var eventChan chan string
 
 // Test cases
 func TestMain(M *testing.M) {
-	os.Setenv("ALARM_IF_RMR", "true")
-	alarmManager = NewAlarmManager("localhost:9093", 500)
-	go alarmManager.Run(false)
+	alarmManager = NewAlarmManager("localhost:9093", 500, false)
+	alarmManager.alertInterval = 20000
+	go alarmManager.Run(false, 5)
 	time.Sleep(time.Duration(10) * time.Second)
 
 	// Wait until RMR is up-and-running
@@ -88,36 +89,48 @@ func TestSetAlarmDefinitions(t *testing.T) {
 	alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
 	alarm8004Definition.EventType = "Processing error"
 	alarm8004Definition.OperationInstructions = "Not defined"
+	alarm8004Definition.RaiseDelay = 0
+	alarm8004Definition.ClearDelay = 0
 
 	var alarm8005Definition alarm.AlarmDefinition
 	alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
 	alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
 	alarm8005Definition.EventType = "Communication error"
 	alarm8005Definition.OperationInstructions = "Not defined"
+	alarm8005Definition.RaiseDelay = 0
+	alarm8005Definition.ClearDelay = 0
 
 	var alarm8006Definition alarm.AlarmDefinition
 	alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
 	alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
 	alarm8006Definition.EventType = "Communication error"
 	alarm8006Definition.OperationInstructions = "Not defined"
+	alarm8006Definition.RaiseDelay = 0
+	alarm8006Definition.ClearDelay = 0
 
 	var alarm8007Definition alarm.AlarmDefinition
 	alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
 	alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
 	alarm8007Definition.EventType = "Communication error"
 	alarm8007Definition.OperationInstructions = "Not defined"
+	alarm8007Definition.RaiseDelay = 0
+	alarm8007Definition.ClearDelay = 0
 
 	var alarm8008Definition alarm.AlarmDefinition
 	alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
 	alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
 	alarm8008Definition.EventType = "storage warning"
 	alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold"
+	alarm8008Definition.RaiseDelay = 0
+	alarm8008Definition.ClearDelay = 0
 
 	var alarm8009Definition alarm.AlarmDefinition
 	alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
 	alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
 	alarm8009Definition.EventType = "storage warning"
 	alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold"
+	alarm8009Definition.RaiseDelay = 0
+	alarm8009Definition.ClearDelay = 0
 
 	pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
 	pbodyEn, _ := json.Marshal(pbodyParams)
@@ -181,6 +194,8 @@ func TestDeleteAlarmDefinitions(t *testing.T) {
 	alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
 	alarm8004Definition.EventType = "Processing error"
 	alarm8004Definition.OperationInstructions = "Not defined"
+	alarm8004Definition.RaiseDelay = 0
+	alarm8004Definition.ClearDelay = 0
 	pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
 	pbodyEn, _ := json.Marshal(pbodyParams)
 	req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
@@ -239,6 +254,7 @@ func TestMultipleAlarmsRaisedSucess(t *testing.T) {
 	b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
 	assert.Nil(t, alarmer.Raise(b), "raise failed")
 
+	time.Sleep(time.Duration(2) * time.Second)
 	VerifyAlarm(t, a, 2)
 	VerifyAlarm(t, b, 2)
 }
@@ -330,10 +346,187 @@ func TestActiveAlarmMaxThresholds(t *testing.T) {
 	ts.Close()
 }
 
+func TestGetPrometheusAlerts(t *testing.T) {
+	time.Sleep(1 * time.Second)
+	xapp.Logger.Info("TestGetPrometheusAlerts")
+	ts := CreatePromAlertSimulator2(t, "GET", "/alerts?active=true&inhibited=true&silenced=true&unprocessed=true")
+
+	commandReady := make(chan bool, 1)
+	command := "cli/alarm-cli"
+	args := []string{"alerts", "--active", "true", "--inhibited", "true", "--silenced", "--unprocessed", "true", "true", "--host", "localhost", "--port", "9093", "flushall"}
+	ExecCLICommand(commandReady, command, args...)
+	<-commandReady
+
+	ts.Close()
+}
+
+func TestDelayedAlarmRaiseAndClear(t *testing.T) {
+	xapp.Logger.Info("TestDelayedAlarmRaiseAndClear")
+
+	activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
+	alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
+
+	// Add new alarm definition
+	var alarm9999Definition alarm.AlarmDefinition
+	alarm9999Definition.AlarmId = 9999
+	alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
+	alarm9999Definition.EventType = "Test type"
+	alarm9999Definition.OperationInstructions = "Not defined"
+	alarm9999Definition.RaiseDelay = 1
+	alarm9999Definition.ClearDelay = 1
+	pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
+	pbodyEn, _ := json.Marshal(pbodyParams)
+	req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
+	handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
+	response := executeRequest(req, handleFunc)
+	checkResponseCode(t, http.StatusOK, response.Code)
+
+	// Verify 9999 alarm definition
+	req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
+	vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
+	req = mux.SetURLVars(req, vars)
+	handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
+	response = executeRequest(req, handleFunc)
+	checkResponseCode(t, http.StatusOK, response.Code)
+
+	ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
+	defer ts.Close()
+
+	// Raise alarm. Posting alert and updating alarm history should be delayed
+	a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
+	assert.Nil(t, alarmer.Raise(a), "raise failed")
+	VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
+
+	// Clear the alarm and check the alarm is removed. Posting alert clear and updating alarm history should be delayed
+	assert.Nil(t, alarmer.Clear(a), "clear failed")
+
+	time.Sleep(time.Duration(2) * time.Second)
+	assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
+	assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+2)
+}
+
+func TestDelayedAlarmRaiseAndClear2(t *testing.T) {
+	xapp.Logger.Info("TestDelayedAlarmRaiseAndClear2")
+
+	activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
+	alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
+
+	ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
+	defer ts.Close()
+
+	// Raise two alarms. The first should be delayed
+	a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
+	assert.Nil(t, alarmer.Raise(a), "raise failed")
+	VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
+
+	b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
+	assert.Nil(t, alarmer.Raise(b), "raise failed")
+	VerifyAlarm(t, b, activeAlarmsBeforeTest+2)
+
+	// Clear two alarms. The first should be delayed. Check the alarms are removed
+	assert.Nil(t, alarmer.Clear(a), "clear failed")
+	assert.Nil(t, alarmer.Clear(b), "clear failed")
+
+	time.Sleep(time.Duration(2) * time.Second)
+	assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
+	assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
+}
+
+func TestDelayedAlarmRaiseAndClear3(t *testing.T) {
+	xapp.Logger.Info("TestDelayedAlarmRaiseAndClear3")
+
+	// Delete exisitng 9999 alarm definition
+	req, _ := http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
+	vars := map[string]string{"alarmId": strconv.FormatUint(9999, 10)}
+	req = mux.SetURLVars(req, vars)
+	handleFunc := http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
+	response := executeRequest(req, handleFunc)
+	checkResponseCode(t, http.StatusOK, response.Code)
+
+	// Add updated 9999 alarm definition
+	var alarm9999Definition alarm.AlarmDefinition
+	alarm9999Definition.AlarmId = 9999
+	alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
+	alarm9999Definition.EventType = "Test type"
+	alarm9999Definition.OperationInstructions = "Not defined"
+	alarm9999Definition.RaiseDelay = 1
+	alarm9999Definition.ClearDelay = 0
+	pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
+	pbodyEn, _ := json.Marshal(pbodyParams)
+	req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
+	handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
+	response = executeRequest(req, handleFunc)
+	checkResponseCode(t, http.StatusOK, response.Code)
+
+	// Verify 9999 alarm definition
+	req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
+	vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
+	req = mux.SetURLVars(req, vars)
+	handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
+	response = executeRequest(req, handleFunc)
+	checkResponseCode(t, http.StatusOK, response.Code)
+
+	activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
+	alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
+
+	ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
+	defer ts.Close()
+
+	// Raise two alarms. The first should be delayed
+	a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
+	assert.Nil(t, alarmer.Raise(a), "raise failed")
+	VerifyAlarm(t, a, activeAlarmsBeforeTest+1)
+
+	b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
+	assert.Nil(t, alarmer.Raise(b), "raise failed")
+	VerifyAlarm(t, b, activeAlarmsBeforeTest+2)
+
+	// Clear two alarms. The first should be delayed. Check the alarms are removed
+	assert.Nil(t, alarmer.Clear(a), "clear failed")
+	assert.Nil(t, alarmer.Clear(b), "clear failed")
+
+	time.Sleep(time.Duration(2) * time.Second)
+	assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
+	assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest+4)
+}
+
+func TestClearExpiredAlarms(t *testing.T) {
+	xapp.Logger.Info("TestClearExpiredAlarms")
+
+	a := alarm.AlarmMessage{
+		Alarm:       alarmer.NewAlarm(8007, alarm.SeverityWarning, "threshold", ""),
+		AlarmAction: alarm.AlarmActionRaise,
+		AlarmTime:   time.Now().UnixNano(),
+	}
+	d := alarm.RICAlarmDefinitions[8007]
+	n := AlarmNotification{a, *d}
+	alarmManager.activeAlarms = make([]AlarmNotification, 0)
+	alarmManager.UpdateActiveAlarmList(&n)
+
+	// Unknown SP
+	a.Alarm.SpecificProblem = 1234
+	assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
+
+	// TTL is 0
+	d.TimeToLive = 0
+	assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
+
+	// TTL not expired
+	a.Alarm.SpecificProblem = 8007
+	d.TimeToLive = 2
+	assert.False(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
+
+	// TTL expired, alarm should be cleared
+	time.Sleep(time.Duration(3) * time.Second)
+	assert.Equal(t, len(alarmManager.activeAlarms), 1)
+	assert.True(t, alarmManager.ClearExpiredAlarms(n, 0, false), "ClearExpiredAlarms failed")
+	assert.Equal(t, len(alarmManager.activeAlarms), 0)
+}
+
 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
 	receivedAlert := waitForEvent()
 
-	assert.Equal(t, len(alarmManager.activeAlarms), expectedCount)
+	assert.Equal(t, expectedCount, len(alarmManager.activeAlarms))
 	_, ok := alarmManager.IsMatchFound(a)
 	assert.True(t, ok)
 
@@ -369,6 +562,33 @@ func CreatePromAlertSimulator(t *testing.T, method, url string, status int, resp
 	return ts
 }
 
+func CreatePromAlertSimulator2(t *testing.T, method, url string) *httptest.Server {
+	l, err := net.Listen("tcp", "localhost:9093")
+	if err != nil {
+		t.Error("Failed to create listener: " + err.Error())
+	}
+	ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, r.Method, method)
+		assert.Equal(t, r.URL.String(), url)
+
+		w.Header().Add("Content-Type", "application/json")
+		w.WriteHeader(200)
+		// Read alerts from file
+		payload, err := readJSONFromFile("../testresources/prometheus-alerts.json")
+		if err != nil {
+			t.Error("Failed to send response: ", err)
+		}
+		_, err = w.Write(payload)
+		if err != nil {
+			t.Error("Failed to send response: " + err.Error())
+		}
+	}))
+	ts.Listener.Close()
+	ts.Listener = l
+	ts.Start()
+	return ts
+}
+
 func waitForEvent() string {
 	receivedAlert := <-eventChan
 	return receivedAlert
@@ -397,3 +617,27 @@ func checkResponseCode(t *testing.T, expected, actual int) bool {
 	}
 	return true
 }
+
+func ExecCLICommand(commandReady chan bool, command string, args ...string) {
+	go func() {
+		xapp.Logger.Info("Giving CLI command")
+		cmd := exec.Command(command, args...)
+		cmd.Dir = "../"
+		output, err := cmd.CombinedOutput()
+		if err != nil {
+			xapp.Logger.Info("CLI command failed out: %s", err)
+		}
+		xapp.Logger.Info("CLI command output: %s", output)
+		commandReady <- true
+		xapp.Logger.Info("CLI command completed")
+	}()
+}
+
+func readJSONFromFile(filename string) ([]byte, error) {
+	file, err := ioutil.ReadFile(filename)
+	if err != nil {
+		err := fmt.Errorf("readJSONFromFile() failed: Error: %v", err)
+		return nil, err
+	}
+	return file, nil
+}