LN0739_FM_FR9: Add configured delay for raise/clear alarm handling
[ric-plt/alarm-go.git] / manager / cmd / manager_test.go
index c048f4b..b75cae5 100755 (executable)
 package main
 
 import (
+       "bytes"
        "encoding/json"
        "fmt"
+       "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
+       "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
+       "github.com/gorilla/mux"
+       "github.com/prometheus/alertmanager/api/v2/models"
        "github.com/stretchr/testify/assert"
        "io"
        "io/ioutil"
@@ -30,13 +35,11 @@ import (
        "net/http"
        "net/http/httptest"
        "os"
+       "os/exec"
+       "strconv"
        "strings"
        "testing"
        "time"
-
-       "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
-       "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
-       "github.com/prometheus/alertmanager/api/v2/models"
 )
 
 var alarmManager *AlarmManager
@@ -45,10 +48,10 @@ var eventChan chan string
 
 // Test cases
 func TestMain(M *testing.M) {
-       os.Setenv("ALARM_IF_RMR", "true")
-       alarmManager = NewAlarmManager("localhost:9093", 500)
+       alarmManager = NewAlarmManager("localhost:9093", 500, false)
+       alarmManager.alertInterval = 20000
        go alarmManager.Run(false)
-       time.Sleep(time.Duration(2) * time.Second)
+       time.Sleep(time.Duration(10) * time.Second)
 
        // Wait until RMR is up-and-running
        for !xapp.Rmr.IsReady() {
@@ -56,13 +59,161 @@ func TestMain(M *testing.M) {
        }
 
        alarmer, _ = alarm.InitAlarm("my-pod", "my-app")
+       alarmManager.alarmClient = alarmer
        time.Sleep(time.Duration(5) * time.Second)
        eventChan = make(chan string)
 
        os.Exit(M.Run())
 }
 
+func TestGetPreDefinedAlarmDefinitions(t *testing.T) {
+       xapp.Logger.Info("TestGetPreDefinedAlarmDefinitions")
+       var alarmDefinition alarm.AlarmDefinition
+       req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
+       vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
+       req = mux.SetURLVars(req, vars)
+       handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
+       response := executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusOK, response.Code)
+       json.NewDecoder(response.Body).Decode(&alarmDefinition)
+       xapp.Logger.Info("alarm definition = %v", alarmDefinition)
+       if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
+               t.Errorf("Incorrect alarm definition")
+       }
+}
+
+func TestSetAlarmDefinitions(t *testing.T) {
+       xapp.Logger.Info("TestSetAlarmDefinitions")
+       var alarm8004Definition alarm.AlarmDefinition
+       alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
+       alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
+       alarm8004Definition.EventType = "Processing error"
+       alarm8004Definition.OperationInstructions = "Not defined"
+       alarm8004Definition.RaiseDelay = 0
+       alarm8004Definition.ClearDelay = 0
+
+       var alarm8005Definition alarm.AlarmDefinition
+       alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
+       alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
+       alarm8005Definition.EventType = "Communication error"
+       alarm8005Definition.OperationInstructions = "Not defined"
+       alarm8005Definition.RaiseDelay = 0
+       alarm8005Definition.ClearDelay = 0
+
+       var alarm8006Definition alarm.AlarmDefinition
+       alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
+       alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
+       alarm8006Definition.EventType = "Communication error"
+       alarm8006Definition.OperationInstructions = "Not defined"
+       alarm8006Definition.RaiseDelay = 0
+       alarm8006Definition.ClearDelay = 0
+
+       var alarm8007Definition alarm.AlarmDefinition
+       alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
+       alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
+       alarm8007Definition.EventType = "Communication error"
+       alarm8007Definition.OperationInstructions = "Not defined"
+       alarm8007Definition.RaiseDelay = 0
+       alarm8007Definition.ClearDelay = 0
+
+       var alarm8008Definition alarm.AlarmDefinition
+       alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
+       alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
+       alarm8008Definition.EventType = "storage warning"
+       alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold"
+       alarm8008Definition.RaiseDelay = 0
+       alarm8008Definition.ClearDelay = 0
+
+       var alarm8009Definition alarm.AlarmDefinition
+       alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
+       alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
+       alarm8009Definition.EventType = "storage warning"
+       alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold"
+       alarm8009Definition.RaiseDelay = 0
+       alarm8009Definition.ClearDelay = 0
+
+       pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
+       pbodyEn, _ := json.Marshal(pbodyParams)
+       req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
+       handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
+       response := executeRequest(req, handleFunc)
+       status := checkResponseCode(t, http.StatusOK, response.Code)
+       xapp.Logger.Info("status = %v", status)
+
+}
+
+func TestGetAlarmDefinitions(t *testing.T) {
+       xapp.Logger.Info("TestGetAlarmDefinitions")
+       var alarmDefinition alarm.AlarmDefinition
+       req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
+       vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
+       req = mux.SetURLVars(req, vars)
+       handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
+       response := executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusOK, response.Code)
+       json.NewDecoder(response.Body).Decode(&alarmDefinition)
+       xapp.Logger.Info("alarm definition = %v", alarmDefinition)
+       if alarmDefinition.AlarmId != alarm.RIC_RT_DISTRIBUTION_FAILED || alarmDefinition.AlarmText != "RIC ROUTING TABLE DISTRIBUTION FAILED" {
+               t.Errorf("Incorrect alarm definition")
+       }
+}
+
+func TestDeleteAlarmDefinitions(t *testing.T) {
+       xapp.Logger.Info("TestDeleteAlarmDefinitions")
+       //Get all
+       var ricAlarmDefinitions RicAlarmDefinitions
+       req, _ := http.NewRequest("GET", "/ric/v1/alarms/define", nil)
+       req = mux.SetURLVars(req, nil)
+       handleFunc := http.HandlerFunc(alarmManager.GetAlarmDefinition)
+       response := executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusOK, response.Code)
+       json.NewDecoder(response.Body).Decode(&ricAlarmDefinitions)
+       for _, alarmDefinition := range ricAlarmDefinitions.AlarmDefinitions {
+               xapp.Logger.Info("alarm definition = %v", *alarmDefinition)
+       }
+
+       //Delete 8004
+       req, _ = http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
+       vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
+       req = mux.SetURLVars(req, vars)
+       handleFunc = http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
+       response = executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusOK, response.Code)
+
+       //Get 8004 fail
+       req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
+       vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
+       req = mux.SetURLVars(req, vars)
+       handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
+       response = executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusBadRequest, response.Code)
+
+       //Set 8004 success
+       var alarm8004Definition alarm.AlarmDefinition
+       alarm8004Definition.AlarmId = alarm.RIC_RT_DISTRIBUTION_FAILED
+       alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
+       alarm8004Definition.EventType = "Processing error"
+       alarm8004Definition.OperationInstructions = "Not defined"
+       alarm8004Definition.RaiseDelay = 0
+       alarm8004Definition.ClearDelay = 0
+       pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
+       pbodyEn, _ := json.Marshal(pbodyParams)
+       req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
+       handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
+       response = executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusOK, response.Code)
+
+       //Get 8004 success
+       req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
+       vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
+       req = mux.SetURLVars(req, vars)
+       handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
+       response = executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusOK, response.Code)
+}
+
 func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
+       xapp.Logger.Info("TestNewAlarmStoredAndPostedSucess")
        ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
        defer ts.Close()
 
@@ -73,6 +224,7 @@ func TestNewAlarmStoredAndPostedSucess(t *testing.T) {
 }
 
 func TestAlarmClearedSucess(t *testing.T) {
+       xapp.Logger.Info("TestAlarmClearedSucess")
        ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
        defer ts.Close()
 
@@ -91,6 +243,7 @@ func TestAlarmClearedSucess(t *testing.T) {
 }
 
 func TestMultipleAlarmsRaisedSucess(t *testing.T) {
+       xapp.Logger.Info("TestMultipleAlarmsRaisedSucess")
        ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
        defer ts.Close()
 
@@ -101,11 +254,13 @@ func TestMultipleAlarmsRaisedSucess(t *testing.T) {
        b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
        assert.Nil(t, alarmer.Raise(b), "raise failed")
 
+       time.Sleep(time.Duration(2) * time.Second)
        VerifyAlarm(t, a, 2)
        VerifyAlarm(t, b, 2)
 }
 
 func TestMultipleAlarmsClearedSucess(t *testing.T) {
+       xapp.Logger.Info("TestMultipleAlarmsClearedSucess")
        ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
        defer ts.Close()
 
@@ -121,6 +276,7 @@ func TestMultipleAlarmsClearedSucess(t *testing.T) {
 }
 
 func TestAlarmsSuppresedSucess(t *testing.T) {
+       xapp.Logger.Info("TestAlarmsSuppresedSucess")
        ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
        defer ts.Close()
 
@@ -130,33 +286,214 @@ func TestAlarmsSuppresedSucess(t *testing.T) {
        assert.Nil(t, alarmer.Raise(a), "raise failed")
 
        VerifyAlarm(t, a, 1)
+       assert.Nil(t, alarmer.Clear(a), "clear failed")
 }
 
 func TestInvalidAlarms(t *testing.T) {
+       xapp.Logger.Info("TestInvalidAlarms")
        a := alarmer.NewAlarm(1111, alarm.SeverityMajor, "Some App data", "eth 0 1")
        assert.Nil(t, alarmer.Raise(a), "raise failed")
        time.Sleep(time.Duration(2) * time.Second)
 }
 
 func TestAlarmHandlingErrorCases(t *testing.T) {
+       xapp.Logger.Info("TestAlarmHandlingErrorCases")
        ok, err := alarmManager.HandleAlarms(&xapp.RMRParams{})
        assert.Equal(t, err.Error(), "unexpected end of JSON input")
        assert.Nil(t, ok, "raise failed")
 }
 
 func TestConsumeUnknownMessage(t *testing.T) {
+       xapp.Logger.Info("TestConsumeUnknownMessage")
        err := alarmManager.Consume(&xapp.RMRParams{})
        assert.Nil(t, err, "raise failed")
 }
 
 func TestStatusCallback(t *testing.T) {
+       xapp.Logger.Info("TestStatusCallback")
        assert.Equal(t, true, alarmManager.StatusCB())
 }
 
+func TestActiveAlarmMaxThresholds(t *testing.T) {
+       xapp.Logger.Info("TestActiveAlarmMaxThresholds")
+       ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
+       alarmManager.maxActiveAlarms = 0
+       alarmManager.maxAlarmHistory = 10
+
+       a := alarmer.NewAlarm(alarm.E2_CONNECTIVITY_LOST_TO_GNODEB, alarm.SeverityCritical, "Some Application data", "eth 0 2")
+       assert.Nil(t, alarmer.Raise(a), "raise failed")
+
+       var alarmConfigParams alarm.AlarmConfigParams
+       req, _ := http.NewRequest("GET", "/ric/v1/alarms/config", nil)
+       req = mux.SetURLVars(req, nil)
+       handleFunc := http.HandlerFunc(alarmManager.GetAlarmConfig)
+       response := executeRequest(req, handleFunc)
+
+       // Check HTTP Status Code
+       checkResponseCode(t, http.StatusOK, response.Code)
+
+       // Decode the json output from handler
+       json.NewDecoder(response.Body).Decode(&alarmConfigParams)
+       if alarmConfigParams.MaxActiveAlarms != 0 || alarmConfigParams.MaxAlarmHistory != 10 {
+               t.Errorf("Incorrect alarm thresholds")
+       }
+
+       time.Sleep(time.Duration(1) * time.Second)
+       alarmManager.maxActiveAlarms = 5000
+       alarmManager.maxAlarmHistory = 20000
+       VerifyAlarm(t, a, 2)
+       VerifyAlarm(t, a, 2)
+       ts.Close()
+}
+
+func TestGetPrometheusAlerts(t *testing.T) {
+       time.Sleep(1 * time.Second)
+       xapp.Logger.Info("TestGetPrometheusAlerts")
+       ts := CreatePromAlertSimulator2(t, "GET", "/alerts?active=true&inhibited=true&silenced=true&unprocessed=true")
+
+       commandReady := make(chan bool, 1)
+       command := "cli/alarm-cli"
+       args := []string{"gapam", "--active", "true", "--inhibited", "true", "--silenced", "--unprocessed", "true", "true", "--host", "localhost", "--port", "9093", "flushall"}
+       ExecCLICommand(commandReady, command, args...)
+       <-commandReady
+
+       ts.Close()
+}
+
+func TestDelayedAlarmRaiseAndClear(t *testing.T) {
+       xapp.Logger.Info("TestDelayedAlarmRaiseAndClear")
+
+       activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
+       alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
+
+       // Add new alarm definition
+       var alarm9999Definition alarm.AlarmDefinition
+       alarm9999Definition.AlarmId = 9999
+       alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
+       alarm9999Definition.EventType = "Test type"
+       alarm9999Definition.OperationInstructions = "Not defined"
+       alarm9999Definition.RaiseDelay = 1
+       alarm9999Definition.ClearDelay = 1
+       pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
+       pbodyEn, _ := json.Marshal(pbodyParams)
+       req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
+       handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
+       response := executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusOK, response.Code)
+
+       // Verify 9999 alarm definition
+       req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
+       vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
+       req = mux.SetURLVars(req, vars)
+       handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
+       response = executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusOK, response.Code)
+
+       ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
+       defer ts.Close()
+
+       // Raise alarm. Posting alert and updating alarm history should be delayed 
+       a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
+       assert.Nil(t, alarmer.Raise(a), "raise failed")
+       VerifyAlarm(t, a, activeAlarmsBeforeTest + 1)
+
+       // Clear the alarm and check the alarm is removed. Posting alert clear and updating alarm history should be delayed
+       assert.Nil(t, alarmer.Clear(a), "clear failed")
+
+       time.Sleep(time.Duration(2) * time.Second)
+       assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
+       assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest + 2)
+}
+
+func TestDelayedAlarmRaiseAndClear2(t *testing.T) {
+       xapp.Logger.Info("TestDelayedAlarmRaiseAndClear2")
+
+       activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
+       alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
+
+       ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
+       defer ts.Close()
+
+       // Raise two alarms. The first should be delayed
+       a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
+       assert.Nil(t, alarmer.Raise(a), "raise failed")
+       VerifyAlarm(t, a, activeAlarmsBeforeTest + 1)
+
+       b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
+       assert.Nil(t, alarmer.Raise(b), "raise failed")
+       VerifyAlarm(t, b, activeAlarmsBeforeTest + 2)
+
+       // Clear two alarms. The first should be delayed. Check the alarms are removed
+       assert.Nil(t, alarmer.Clear(a), "clear failed")
+       assert.Nil(t, alarmer.Clear(b), "clear failed")
+
+       time.Sleep(time.Duration(2) * time.Second)
+       assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
+       assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest + 4)
+}
+
+func TestDelayedAlarmRaiseAndClear3(t *testing.T) {
+       xapp.Logger.Info("TestDelayedAlarmRaiseAndClear3")
+
+       // Delete exisitng 9999 alarm definition
+       req, _ := http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
+       vars := map[string]string{"alarmId": strconv.FormatUint(9999, 10)}
+       req = mux.SetURLVars(req, vars)
+       handleFunc := http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
+       response := executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusOK, response.Code)
+
+       // Add updated 9999 alarm definition
+       var alarm9999Definition alarm.AlarmDefinition
+       alarm9999Definition.AlarmId = 9999
+       alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
+       alarm9999Definition.EventType = "Test type"
+       alarm9999Definition.OperationInstructions = "Not defined"
+       alarm9999Definition.RaiseDelay = 1
+       alarm9999Definition.ClearDelay = 0
+       pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
+       pbodyEn, _ := json.Marshal(pbodyParams)
+       req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
+       handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
+       response = executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusOK, response.Code)
+
+       // Verify 9999 alarm definition
+       req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
+       vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
+       req = mux.SetURLVars(req, vars)
+       handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
+       response = executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusOK, response.Code)
+
+       activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
+       alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
+
+       ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
+       defer ts.Close()
+
+       // Raise two alarms. The first should be delayed
+       a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
+       assert.Nil(t, alarmer.Raise(a), "raise failed")
+       VerifyAlarm(t, a, activeAlarmsBeforeTest + 1)
+
+       b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
+       assert.Nil(t, alarmer.Raise(b), "raise failed")
+       VerifyAlarm(t, b, activeAlarmsBeforeTest + 2)
+
+       // Clear two alarms. The first should be delayed. Check the alarms are removed
+       assert.Nil(t, alarmer.Clear(a), "clear failed")
+       assert.Nil(t, alarmer.Clear(b), "clear failed")
+
+       time.Sleep(time.Duration(2) * time.Second)
+       assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
+       assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest + 4)
+}
+
 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
        receivedAlert := waitForEvent()
 
-       assert.Equal(t, len(alarmManager.activeAlarms), expectedCount)
+       assert.Equal(t, expectedCount, len(alarmManager.activeAlarms))
        _, ok := alarmManager.IsMatchFound(a)
        assert.True(t, ok)
 
@@ -192,6 +529,33 @@ func CreatePromAlertSimulator(t *testing.T, method, url string, status int, resp
        return ts
 }
 
+func CreatePromAlertSimulator2(t *testing.T, method, url string) *httptest.Server {
+       l, err := net.Listen("tcp", "localhost:9093")
+       if err != nil {
+               t.Error("Failed to create listener: " + err.Error())
+       }
+       ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+               assert.Equal(t, r.Method, method)
+               assert.Equal(t, r.URL.String(), url)
+
+               w.Header().Add("Content-Type", "application/json")
+               w.WriteHeader(200)
+               // Read alerts from file
+               payload, err := readJSONFromFile("../testresources/prometheus-alerts.json")
+               if err != nil {
+                       t.Error("Failed to send response: ", err)
+               }
+               _, err = w.Write(payload)
+               if err != nil {
+                       t.Error("Failed to send response: " + err.Error())
+               }
+       }))
+       ts.Listener.Close()
+       ts.Listener = l
+       ts.Start()
+       return ts
+}
+
 func waitForEvent() string {
        receivedAlert := <-eventChan
        return receivedAlert
@@ -204,3 +568,43 @@ func fireEvent(t *testing.T, body io.ReadCloser) {
 
        eventChan <- fmt.Sprintf("%s", reqBody)
 }
+
+func executeRequest(req *http.Request, handleR http.HandlerFunc) *httptest.ResponseRecorder {
+       rr := httptest.NewRecorder()
+
+       handleR.ServeHTTP(rr, req)
+
+       return rr
+}
+
+func checkResponseCode(t *testing.T, expected, actual int) bool {
+       if expected != actual {
+               t.Errorf("Expected response code %d. Got %d\n", expected, actual)
+               return false
+       }
+       return true
+}
+
+func ExecCLICommand(commandReady chan bool, command string, args ...string) {
+       go func() {
+               xapp.Logger.Info("Giving CLI command")
+               cmd := exec.Command(command, args...)
+               cmd.Dir = "../"
+               output, err := cmd.CombinedOutput()
+               if err != nil {
+                       xapp.Logger.Info("CLI command failed out: %s", err)
+               }
+               xapp.Logger.Info("CLI command output: %s", output)
+               commandReady <- true
+               xapp.Logger.Info("CLI command completed")
+       }()
+}
+
+func readJSONFromFile(filename string) ([]byte, error) {
+       file, err := ioutil.ReadFile(filename)
+       if err != nil {
+               err := fmt.Errorf("readJSONFromFile() failed: Error: %v", err)
+               return nil, err
+       }
+       return file, nil
+}