LN0739_FM_FR9: Add configured delay for raise/clear alarm handling 37/4937/2
authorAnssi Mannila <anssi.mannila@nokia.com>
Thu, 29 Oct 2020 08:01:00 +0000 (10:01 +0200)
committerAnssi Mannila <anssi.mannila@nokia.com>
Thu, 29 Oct 2020 08:33:19 +0000 (10:33 +0200)
Change-Id: Iec0b53a825fdbca2221672910f517aaeaba2b44c
Signed-off-by: Anssi Mannila <anssi.mannila@nokia.com>
alarm/types.go
build/container-tag.yaml
cli/alarm-cli.go
definitions/alarm-definition.json
docs/user-guide.rst
go.mod
manager/cmd/manager.go
manager/cmd/manager_test.go
manager/cmd/restapi.go
testresources/perf-alarm-definition-delay.json [new file with mode: 0755]
testresources/perf-alarm-definition.json

index 2aaa84b..200d435 100755 (executable)
@@ -151,7 +151,9 @@ type AlarmDefinition struct {
        AlarmId               int    `json:"alarmId"`
        AlarmText             string `json:"alarmText"`
        EventType             string `json:"eventType"`
-       OperationInstructions string `json:"operationinstructions"`
+       OperationInstructions string `json:"operationInstructions"`
+       RaiseDelay            int    `json:"raiseDelay"`
+       ClearDelay            int    `json:"clearDelay"`
 }
 
 var RICAlarmDefinitions map[int]*AlarmDefinition
index 4f49c1f..599041f 100644 (file)
@@ -2,4 +2,4 @@
 # By default this file is in the docker build directory,
 # but the location can configured in the JJB template.
 ---
-tag: 0.5.6
+tag: 0.5.7
index 9a18015..b213ec6 100755 (executable)
@@ -142,6 +142,8 @@ func main() {
                AddFlag("atx", "alarm text", commando.String, nil).
                AddFlag("ety", "event type", commando.String, nil).
                AddFlag("oin", "operation instructions", commando.String, nil).
+               AddFlag("rad", "Raise alarm delay", commando.Int, 0).
+               AddFlag("cad", "Clear alarm delay", commando.Int, 0).
                AddFlag("host", "Alarm manager host address", commando.String, alarmManagerHost).
                AddFlag("port", "Alarm manager host address", commando.String, "8080").
                SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) {
@@ -292,15 +294,18 @@ func displayAlarms(alarms []AlarmNotification, isHistory bool) {
        }
 
        for _, a := range alarms {
+               // Do not show alarm before raiseDelay has elapsed
                alarmTime := time.Unix(0, a.AlarmTime).Format("02/01/2006, 15:04:05")
                if isHistory {
                        t.AppendRows([]table.Row{
                                {a.AlarmId, a.SpecificProblem, a.ManagedObjectId, a.ApplicationId, a.IdentifyingInfo, a.PerceivedSeverity, a.AdditionalInfo, a.AlarmAction, alarmTime},
                        })
                } else {
-                       t.AppendRows([]table.Row{
-                               {a.AlarmId, a.SpecificProblem, a.ManagedObjectId, a.ApplicationId, a.IdentifyingInfo, a.PerceivedSeverity, a.AdditionalInfo, alarmTime},
-                       })
+                       if a.AlarmDefinition.RaiseDelay == 0 {
+                               t.AppendRows([]table.Row{
+                                       {a.AlarmId, a.SpecificProblem, a.ManagedObjectId, a.ApplicationId, a.IdentifyingInfo, a.PerceivedSeverity, a.AdditionalInfo, alarmTime},
+                               })
+                       }
                }
        }
 
@@ -336,6 +341,9 @@ func postAlarmDefinition(flags map[string]commando.FlagValue) {
        alarmtxt, _ := flags["atx"].GetString()
        etype, _ := flags["ety"].GetString()
        operation, _ := flags["oin"].GetString()
+       raiseDelay, _ := flags["rad"].GetInt()
+       clearDelay, _ := flags["cad"].GetInt()
+
        targetUrl := fmt.Sprintf("http://%s:%s/ric/v1/alarms/define", host, port)
 
        var alarmdefinition alarm.AlarmDefinition
@@ -343,6 +351,8 @@ func postAlarmDefinition(flags map[string]commando.FlagValue) {
        alarmdefinition.AlarmText = alarmtxt
        alarmdefinition.EventType = etype
        alarmdefinition.OperationInstructions = operation
+       alarmdefinition.RaiseDelay = raiseDelay
+       alarmdefinition.ClearDelay = clearDelay
 
        m := CliAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarmdefinition}}
        jsonData, err := json.Marshal(m)
@@ -426,7 +436,6 @@ func conductperformancetest(flags map[string]commando.FlagValue) {
        } else {
                fmt.Println("reading performance alarm definitions from json file failed ")
        }
-
 }
 
 func peakPerformanceTest(flags map[string]commando.FlagValue) {
@@ -466,7 +475,10 @@ func enduranceTest(flags map[string]commando.FlagValue) {
 }
 
 func readPerfAlarmObjectFromJson() error {
+
        filename := os.Getenv("PERF_OBJ_FILE")
+       fmt.Printf("readPerfAlarmObjectFromJson: filename = %s\n", filename)
+
        file, err := ioutil.ReadFile(filename)
        if err == nil {
                data := RicPerfAlarmObjects{}
@@ -483,18 +495,22 @@ func readPerfAlarmObjectFromJson() error {
                                CLIPerfAlarmObjects[alarmObject.SpecificProblem] = ricAlarmObject
                        }
                } else {
-                       fmt.Println("readPerfAlarmObjectFromJson: json.Unmarshal failed with error ", err)
+                       fmt.Println("readPerfAlarmObjectFromJson: json.Unmarshal failed with error: ", err)
                        return err
                }
        } else {
-               fmt.Println("readPerfAlarmObjectFromJson: ioutil.ReadFile failed with error ", err)
+               fmt.Printf("readPerfAlarmObjectFromJson: ioutil.ReadFile failed with error: %v, filename: %s\n", err, filename)
+               fmt.Printf("readPerfAlarmObjectFromJson: current directory: %s\n", getCurrentDirectory())
                return err
        }
        return nil
 }
 
 func readPerfAlarmDefinitionFromJson() error {
+
        filename := os.Getenv("PERF_DEF_FILE")
+       fmt.Printf("ReadPerfAlarmDefinitionFromJson: filename = %s\n", filename)
+
        file, err := ioutil.ReadFile(filename)
        if err == nil {
                data := CliAlarmDefinitions{}
@@ -511,6 +527,8 @@ func readPerfAlarmDefinitionFromJson() error {
                                        ricAlarmDefintion.AlarmText = alarmDefinition.AlarmText
                                        ricAlarmDefintion.EventType = alarmDefinition.EventType
                                        ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions
+                                       ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay
+                                       ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay
                                        CliPerfAlarmDefinitions.AlarmDefinitions = append(CliPerfAlarmDefinitions.AlarmDefinitions, ricAlarmDefintion)
                                }
                        }
@@ -519,7 +537,9 @@ func readPerfAlarmDefinitionFromJson() error {
                        return err
                }
        } else {
-               fmt.Println("ReadPerfAlarmDefinitionFromJson: ioutil.ReadFile failed with error: ", err)
+               fmt.Printf("ReadPerfAlarmDefinitionFromJson: ioutil.ReadFile failed with error: %v, filename: %s\n", err, filename)
+               fmt.Printf("ReadPerfAlarmDefinitionFromJson: current directory: %s\n", getCurrentDirectory())
+       
                return err
        }
        return nil
@@ -681,3 +701,11 @@ func newAlertManagerClient(amAddress string, amBaseUrl string, amSchemes []strin
        cr := clientruntime.New(amAddress, amBaseUrl, amSchemes)
        return client.New(cr, strfmt.Default)
 }
+
+func getCurrentDirectory() string {
+       dir, err := os.Getwd()
+       if err != nil {
+               fmt.Println(err)
+       }
+       return dir
+}
\ No newline at end of file
index eebf842..ce32693 100755 (executable)
@@ -4,37 +4,49 @@
             "alarmId" : 8004,
             "alarmText" : "RIC ROUTING TABLE DISTRIBUTION FAILED",
             "eventType" : "processingError",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 8005,
             "alarmText" : "TCP CONNECTIVITY LOST TO DBAAS",
             "eventType" : "communication",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 8006,
             "alarmText" : "E2 CONNECTIVITY LOST TO G-NODEB",
             "eventType" : "communication",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 8007,
             "alarmText" : "E2 CONNECTIVITY LOST TO E-NODEB",
             "eventType" : "communication",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 8008,
             "alarmText" : "ACTIVE ALARM EXCEED MAX THRESHOLD",
             "eventType" : "equipment",
-            "operationInstructions" : "Clear alarms or raise threshold"
+            "operationInstructions" : "Clear alarms or raise threshold",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 8009,
             "alarmText" : "ALARM HISTORY EXCEED MAX THRESHOLD",
             "eventType" : "equipment",
-            "operationInstructions" : "Clear alarms or raise threshold"
+            "operationInstructions" : "Clear alarms or raise threshold",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         }
    ]
 }
index e531f7c..bc08e13 100755 (executable)
@@ -147,6 +147,8 @@ CLI commands can have some of the following parameters
  --atx         Alarm text string, example string: E2 CONNECTIVITY LOST TO E-NODEB
  --ety         Event type string, example string: Communication error
  --oin         Operation instructions string, example string: Not defined
+ --rad         Raise alarm delay in seconds. Default value = 0
+ --cad         Clear alarm delay in seconds. Default value = 0
  --prf         Performance profile id, possible values: 1 = peak performance test or 2 = endurance test
  --nal         Number of alarms, example value: 50
  --aps         Alarms per second, example value: 1
@@ -226,11 +228,11 @@ CLI command examples:
 
  .. code-block:: none
 
-  Syntax: cli/alarm-cli define --aid 8007 --atx "E2 CONNECTIVITY LOST TO E-NODEB" --ety "Communication error" --oin "Not defined" [--host] [--port]
+  Syntax: cli/alarm-cli define --aid 8007 --atx "E2 CONNECTIVITY LOST TO E-NODEB" --ety "Communication error" --oin "Not defined" [--rad] [--cad] [--host] [--port]
 
-  Example: cli/alarm-cli define --aid 8007 --atx "E2 CONNECTIVITY LOST TO E-NODEB" --ety "Communication error" --oin "Not defined"
+  Example: cli/alarm-cli define --aid 8007 --atx "E2 CONNECTIVITY LOST TO E-NODEB" --ety "Communication error" --oin "Not defined" --rad 0 --cad 0
 
-  Example: cli/alarm-cli define --aid 8007 --atx "E2 CONNECTIVITY LOST TO E-NODEB" --ety "Communication error" --oin "Not defined" --host localhost --port 8080
+  Example: cli/alarm-cli define --aid 8007 --atx "E2 CONNECTIVITY LOST TO E-NODEB" --ety "Communication error" --oin "Not defined" --rad 0 --cad 0 --host localhost --port 8080
 
  Delete existing alarm definition:
 
@@ -319,11 +321,11 @@ Below are examples for REST interface. Curl tool is used to send REST commands.
 
  Add one new alarm definition:
 
-   Example: curl -X POST "http://localhost:8080/ric/v1/alarms/define" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"alarmdefinitions\": [{\"alarmId\": 8007, \"alarmText\": \"E2 CONNECTIVITY LOST TO E-NODEB\", \"eventtype\": \"Communication error\", \"operationinstructions\": \"Not defined\"}]}"
+   Example: curl -X POST "http://localhost:8080/ric/v1/alarms/define" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"alarmdefinitions\": [{\"alarmId\": 8007, \"alarmText\": \"E2 CONNECTIVITY LOST TO E-NODEB\", \"eventtype\": \"Communication error\", \"operationinstructions\": \"Not defined\, \"raiseDelay\": 1, \"clearDelay\": 1"}]}"
 
  Add two new alarm definitions:
 
-   Example: curl -X POST "http://localhost:8080/ric/v1/alarms/define" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"alarmdefinitions\": [{\"alarmId\": 8007, \"alarmText\": \"E2 CONNECTIVITY LOST TO E-NODEB\", \"eventtype\": \"Communication error\", \"operationinstructions\": \"Not defined\"},{\"alarmId\": 8008, \"alarmText\": \"ACTIVE ALARM EXCEED MAX THRESHOLD\", \"eventtype\": \"storage warning\", \"operationinstructions\": \"Clear alarms or raise threshold\"}]}"
+   Example: curl -X POST "http://localhost:8080/ric/v1/alarms/define" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"alarmdefinitions\": [{\"alarmId\": 8007, \"alarmText\": \"E2 CONNECTIVITY LOST TO E-NODEB\", \"eventtype\": \"Communication error\", \"operationinstructions\": \"Not defined\, \"raiseDelay\": 0, \"clearDelay\": 0"},{\"alarmId\": 8008, \"alarmText\": \"ACTIVE ALARM EXCEED MAX THRESHOLD\", \"eventtype\": \"storage warning\", \"operationinstructions\": \"Clear alarms or raise threshold\", \"raiseDelay\": 0, \"clearDelay\": 0}]}"
 
  Delete one existing alarm definition:
 
diff --git a/go.mod b/go.mod
index 0fb43fa..b6ae937 100644 (file)
--- a/go.mod
+++ b/go.mod
@@ -20,8 +20,6 @@ require (
        github.com/gorilla/mux v1.7.1
        github.com/jedib0t/go-pretty v4.3.0+incompatible
        github.com/mattn/go-runewidth v0.0.9 // indirect
-       github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-       github.com/modern-go/reflect2 v1.0.1 // indirect
        github.com/prometheus/alertmanager v0.20.0
        github.com/spf13/viper v1.6.2
        github.com/stretchr/testify v1.5.1
index 2b5af17..f7ad81e 100755 (executable)
@@ -24,6 +24,11 @@ import (
        "bytes"
        "encoding/json"
        "fmt"
+       "io/ioutil"
+       "net/http"
+       "os"
+       "time"
+
        "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
        app "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
        clientruntime "github.com/go-openapi/runtime/client"
@@ -32,10 +37,6 @@ import (
        "github.com/prometheus/alertmanager/api/v2/client/alert"
        "github.com/prometheus/alertmanager/api/v2/models"
        "github.com/spf13/viper"
-       "io/ioutil"
-       "net/http"
-       "os"
-       "time"
 )
 
 func (a *AlarmManager) StartAlertTimer() {
@@ -78,15 +79,16 @@ func (a *AlarmManager) HandleAlarms(rp *app.RMRParams) (*alert.PostAlertsOK, err
 
 func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
        a.mutex.Lock()
-
-       if _, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok {
+       alarmDef := &alarm.AlarmDefinition{}
+       var ok bool
+       if alarmDef, ok = alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok {
                app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem)
                a.mutex.Unlock()
                return nil, nil
        }
 
-       // Suppress duplicate alarms
        idx, found := a.IsMatchFound(m.Alarm)
+       // Suppress duplicate alarms
        if found && m.AlarmAction == alarm.AlarmActionRaise {
                app.Logger.Info("Duplicate alarm found, suppressing ...")
                if m.PerceivedSeverity == a.activeAlarms[idx].PerceivedSeverity {
@@ -102,24 +104,9 @@ func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK,
        // Clear alarm if found from active alarm list
        if m.AlarmAction == alarm.AlarmActionClear {
                if found {
-                       a.UpdateAlarmFields(a.activeAlarms[idx].AlarmId, m)
-                       a.alarmHistory = append(a.alarmHistory, *m)
-                       a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
-                       if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
-                               app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
-                               a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
-                       }
-
-                       if a.exceededActiveAlarmOn && m.Alarm.SpecificProblem == alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD {
-                               a.exceededActiveAlarmOn = false
-                       }
-
-                       if a.exceededAlarmHistoryOn && m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD {
-                               a.exceededAlarmHistoryOn = false
+                       if a.ProcessClearAlarm(m, alarmDef, idx) == false {
+                               return nil, nil
                        }
-
-                       a.WriteAlarmInfoToPersistentVolume()
-
                        if a.postClear {
                                a.mutex.Unlock()
 
@@ -138,11 +125,9 @@ func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK,
 
        // New alarm -> update active alarms and post to Alert Manager
        if m.AlarmAction == alarm.AlarmActionRaise {
-               a.UpdateAlarmFields(a.GenerateAlarmId(), m)
-               a.UpdateAlarmLists(m)
-               a.WriteAlarmInfoToPersistentVolume()
-               a.mutex.Unlock()
-
+               if a.ProcessRaiseAlarm(m, alarmDef) == false {
+                       return nil, nil
+               }
                // Send alarm notification to NOMA, if enabled
                if app.Config.GetBool("controls.noma.enabled") {
                        return a.PostAlarm(m)
@@ -154,6 +139,75 @@ func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK,
        return nil, nil
 }
 
+func (a *AlarmManager)ProcessRaiseAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition) bool {
+       app.Logger.Debug("Raise alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m)
+       // RaiseDelay > 0 in an alarm object in active alarm table indicates that raise delay is still ongoing for the alarm
+       m.AlarmDefinition.RaiseDelay = alarmDef.RaiseDelay
+       a.UpdateAlarmFields(a.GenerateAlarmId(), m)
+       a.UpdateActiveAlarmList(m)
+       a.mutex.Unlock()
+       if alarmDef.RaiseDelay > 0 {
+               timerDelay(alarmDef.RaiseDelay)
+               a.mutex.Lock()
+               // Alarm may have been deleted from active alarms table during delay or table index may have changed
+               idx, found := a.IsMatchFound(m.Alarm)
+               if found {
+                       // Alarm is not showed in active alarms or alarm history via CLI before RaiseDelay has elapsed, i.e the value is 0
+                       a.activeAlarms[idx].AlarmDefinition.RaiseDelay = 0
+                       app.Logger.Debug("Raise after delay alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m)
+                       a.mutex.Unlock()
+               } else {
+                       app.Logger.Debug("Alarm deleted during raise delay. AlarmNotification = %v", *m)
+                       a.mutex.Unlock()
+                       return false
+               }
+       }
+       m.AlarmDefinition.RaiseDelay = 0
+       a.UpdateAlarmHistoryList(m)
+       a.WriteAlarmInfoToPersistentVolume()    
+       return true
+}
+
+func (a *AlarmManager)ProcessClearAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition, idx int) bool {
+       app.Logger.Debug("Clear alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m)
+       if alarmDef.ClearDelay > 0 {
+               a.mutex.Unlock()
+               timerDelay(alarmDef.ClearDelay)
+               app.Logger.Debug("Clear after delay alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m)
+               a.mutex.Lock()
+               // Another alarm clear may have happened during delay and active alarms table index changed
+               var found bool
+               idx, found = a.IsMatchFound(m.Alarm)
+               if !found {
+                       app.Logger.Debug("Alarm not anymore in the active alarms table. AlarmNotification = %v", *m)
+                       a.mutex.Unlock()
+                       return false
+               }
+       }
+       a.UpdateAlarmFields(a.activeAlarms[idx].AlarmId, m)
+       a.alarmHistory = append(a.alarmHistory, *m)
+       a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
+       if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
+               app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
+               a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
+       }
+
+       if a.exceededActiveAlarmOn && m.Alarm.SpecificProblem == alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD {
+               a.exceededActiveAlarmOn = false
+       }
+
+       if a.exceededAlarmHistoryOn && m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD {
+               a.exceededAlarmHistoryOn = false
+       }
+       a.WriteAlarmInfoToPersistentVolume()    
+       return true
+}
+
+func timerDelay(delay int) {
+       timer := time.NewTimer(time.Duration(delay) * time.Second)
+       <-timer.C
+}
+
 func (a *AlarmManager) IsMatchFound(newAlarm alarm.Alarm) (int, bool) {
        for i, m := range a.activeAlarms {
                if m.ManagedObjectId == newAlarm.ManagedObjectId && m.ApplicationId == newAlarm.ApplicationId &&
@@ -198,21 +252,29 @@ func (a *AlarmManager) GenerateThresholdAlarm(sp int, data string) bool {
        return true
 }
 
-func (a *AlarmManager) UpdateAlarmLists(newAlarm *AlarmNotification) {
+func (a *AlarmManager) UpdateActiveAlarmList(newAlarm *AlarmNotification) {
        /* If maximum number of active alarms is reached, an error log writing is made, and new alarm indicating the problem is raised.
-          The attempt to raise the alarm next time will be supressed when found as duplicate. */
+          The attempt to raise the alarm next time will be suppressed when found as duplicate. */
        if (len(a.activeAlarms) >= a.maxActiveAlarms) && (a.exceededActiveAlarmOn == false) {
                app.Logger.Warn("active alarm count exceeded maxActiveAlarms threshold")
                a.exceededActiveAlarmOn = a.GenerateThresholdAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, "active")
        }
 
+       // @todo: For now just keep the  active alarms in-memory. Use SDL later for persistence
+       a.activeAlarms = append(a.activeAlarms, *newAlarm)
+}
+
+func (a *AlarmManager) UpdateAlarmHistoryList(newAlarm *AlarmNotification) {
+       /* If maximum number of events in alarm history is reached, an error log writing is made,
+          and new alarm indicating the problem is raised. The attempt to add new event time will
+          be suppressed */
+
        if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
                app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
                a.exceededAlarmHistoryOn = a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
        }
 
-       // @todo: For now just keep the alarms (both active and history) in-memory. Use SDL later for persistence
-       a.activeAlarms = append(a.activeAlarms, *newAlarm)
+       // @todo: For now just keep the alarms history in-memory. Use SDL later for persistence
        a.alarmHistory = append(a.alarmHistory, *newAlarm)
 }
 
@@ -322,6 +384,8 @@ func (a *AlarmManager) ReadAlarmDefinitionFromJson() {
                                        ricAlarmDefintion.AlarmText = alarmDefinition.AlarmText
                                        ricAlarmDefintion.EventType = alarmDefinition.EventType
                                        ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions
+                                       ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay
+                                       ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay
                                        alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion
                                }
                        }
index 2589a91..b75cae5 100755 (executable)
@@ -89,36 +89,48 @@ func TestSetAlarmDefinitions(t *testing.T) {
        alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
        alarm8004Definition.EventType = "Processing error"
        alarm8004Definition.OperationInstructions = "Not defined"
+       alarm8004Definition.RaiseDelay = 0
+       alarm8004Definition.ClearDelay = 0
 
        var alarm8005Definition alarm.AlarmDefinition
        alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS
        alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS"
        alarm8005Definition.EventType = "Communication error"
        alarm8005Definition.OperationInstructions = "Not defined"
+       alarm8005Definition.RaiseDelay = 0
+       alarm8005Definition.ClearDelay = 0
 
        var alarm8006Definition alarm.AlarmDefinition
        alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB
        alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB"
        alarm8006Definition.EventType = "Communication error"
        alarm8006Definition.OperationInstructions = "Not defined"
+       alarm8006Definition.RaiseDelay = 0
+       alarm8006Definition.ClearDelay = 0
 
        var alarm8007Definition alarm.AlarmDefinition
        alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB
        alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB"
        alarm8007Definition.EventType = "Communication error"
        alarm8007Definition.OperationInstructions = "Not defined"
+       alarm8007Definition.RaiseDelay = 0
+       alarm8007Definition.ClearDelay = 0
 
        var alarm8008Definition alarm.AlarmDefinition
        alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD
        alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD"
        alarm8008Definition.EventType = "storage warning"
        alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold"
+       alarm8008Definition.RaiseDelay = 0
+       alarm8008Definition.ClearDelay = 0
 
        var alarm8009Definition alarm.AlarmDefinition
        alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD
        alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD"
        alarm8009Definition.EventType = "storage warning"
        alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold"
+       alarm8009Definition.RaiseDelay = 0
+       alarm8009Definition.ClearDelay = 0
 
        pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}}
        pbodyEn, _ := json.Marshal(pbodyParams)
@@ -182,6 +194,8 @@ func TestDeleteAlarmDefinitions(t *testing.T) {
        alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED"
        alarm8004Definition.EventType = "Processing error"
        alarm8004Definition.OperationInstructions = "Not defined"
+       alarm8004Definition.RaiseDelay = 0
+       alarm8004Definition.ClearDelay = 0
        pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}}
        pbodyEn, _ := json.Marshal(pbodyParams)
        req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
@@ -346,6 +360,136 @@ func TestGetPrometheusAlerts(t *testing.T) {
        ts.Close()
 }
 
+func TestDelayedAlarmRaiseAndClear(t *testing.T) {
+       xapp.Logger.Info("TestDelayedAlarmRaiseAndClear")
+
+       activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
+       alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
+
+       // Add new alarm definition
+       var alarm9999Definition alarm.AlarmDefinition
+       alarm9999Definition.AlarmId = 9999
+       alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
+       alarm9999Definition.EventType = "Test type"
+       alarm9999Definition.OperationInstructions = "Not defined"
+       alarm9999Definition.RaiseDelay = 1
+       alarm9999Definition.ClearDelay = 1
+       pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
+       pbodyEn, _ := json.Marshal(pbodyParams)
+       req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
+       handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition)
+       response := executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusOK, response.Code)
+
+       // Verify 9999 alarm definition
+       req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
+       vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
+       req = mux.SetURLVars(req, vars)
+       handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
+       response = executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusOK, response.Code)
+
+       ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
+       defer ts.Close()
+
+       // Raise alarm. Posting alert and updating alarm history should be delayed 
+       a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
+       assert.Nil(t, alarmer.Raise(a), "raise failed")
+       VerifyAlarm(t, a, activeAlarmsBeforeTest + 1)
+
+       // Clear the alarm and check the alarm is removed. Posting alert clear and updating alarm history should be delayed
+       assert.Nil(t, alarmer.Clear(a), "clear failed")
+
+       time.Sleep(time.Duration(2) * time.Second)
+       assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
+       assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest + 2)
+}
+
+func TestDelayedAlarmRaiseAndClear2(t *testing.T) {
+       xapp.Logger.Info("TestDelayedAlarmRaiseAndClear2")
+
+       activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
+       alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
+
+       ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
+       defer ts.Close()
+
+       // Raise two alarms. The first should be delayed
+       a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
+       assert.Nil(t, alarmer.Raise(a), "raise failed")
+       VerifyAlarm(t, a, activeAlarmsBeforeTest + 1)
+
+       b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
+       assert.Nil(t, alarmer.Raise(b), "raise failed")
+       VerifyAlarm(t, b, activeAlarmsBeforeTest + 2)
+
+       // Clear two alarms. The first should be delayed. Check the alarms are removed
+       assert.Nil(t, alarmer.Clear(a), "clear failed")
+       assert.Nil(t, alarmer.Clear(b), "clear failed")
+
+       time.Sleep(time.Duration(2) * time.Second)
+       assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
+       assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest + 4)
+}
+
+func TestDelayedAlarmRaiseAndClear3(t *testing.T) {
+       xapp.Logger.Info("TestDelayedAlarmRaiseAndClear3")
+
+       // Delete exisitng 9999 alarm definition
+       req, _ := http.NewRequest("DELETE", "/ric/v1/alarms/define", nil)
+       vars := map[string]string{"alarmId": strconv.FormatUint(9999, 10)}
+       req = mux.SetURLVars(req, vars)
+       handleFunc := http.HandlerFunc(alarmManager.DeleteAlarmDefinition)
+       response := executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusOK, response.Code)
+
+       // Add updated 9999 alarm definition
+       var alarm9999Definition alarm.AlarmDefinition
+       alarm9999Definition.AlarmId = 9999
+       alarm9999Definition.AlarmText = "DELAYED TEST ALARM"
+       alarm9999Definition.EventType = "Test type"
+       alarm9999Definition.OperationInstructions = "Not defined"
+       alarm9999Definition.RaiseDelay = 1
+       alarm9999Definition.ClearDelay = 0
+       pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}}
+       pbodyEn, _ := json.Marshal(pbodyParams)
+       req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn))
+       handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition)
+       response = executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusOK, response.Code)
+
+       // Verify 9999 alarm definition
+       req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil)
+       vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)}
+       req = mux.SetURLVars(req, vars)
+       handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition)
+       response = executeRequest(req, handleFunc)
+       checkResponseCode(t, http.StatusOK, response.Code)
+
+       activeAlarmsBeforeTest := len(alarmManager.activeAlarms)
+       alarmHistoryBeforeTest := len(alarmManager.alarmHistory)
+
+       ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{})
+       defer ts.Close()
+
+       // Raise two alarms. The first should be delayed
+       a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1")
+       assert.Nil(t, alarmer.Raise(a), "raise failed")
+       VerifyAlarm(t, a, activeAlarmsBeforeTest + 1)
+
+       b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1")
+       assert.Nil(t, alarmer.Raise(b), "raise failed")
+       VerifyAlarm(t, b, activeAlarmsBeforeTest + 2)
+
+       // Clear two alarms. The first should be delayed. Check the alarms are removed
+       assert.Nil(t, alarmer.Clear(a), "clear failed")
+       assert.Nil(t, alarmer.Clear(b), "clear failed")
+
+       time.Sleep(time.Duration(2) * time.Second)
+       assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest)
+       assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest + 4)
+}
+
 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
        receivedAlert := waitForEvent()
 
index 412c71c..b8946df 100755 (executable)
@@ -95,6 +95,8 @@ func (a *AlarmManager) SetAlarmDefinition(w http.ResponseWriter, r *http.Request
                        ricAlarmDefintion.AlarmText = alarmDefinition.AlarmText
                        ricAlarmDefintion.EventType = alarmDefinition.EventType
                        ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions
+                       ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay
+                       ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay
                        alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion
                        app.Logger.Debug("POST - alarm definition added for alarm id %v", alarmDefinition.AlarmId)
                }
diff --git a/testresources/perf-alarm-definition-delay.json b/testresources/perf-alarm-definition-delay.json
new file mode 100755 (executable)
index 0000000..e766d74
--- /dev/null
@@ -0,0 +1,404 @@
+{
+    "alarmdefinitions" : [
+        {
+            "alarmId" : 1001,
+            "alarmText" : "PERFORMANCE TEST ALARM 1",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 1,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1002,
+            "alarmText" : "PERFORMANCE TEST ALARM 2",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1003,
+            "alarmText" : "PERFORMANCE TEST ALARM 3",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1004,
+            "alarmText" : "PERFORMANCE TEST ALARM 4",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 2,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1005,
+            "alarmText" : "PERFORMANCE TEST ALARM 5",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1006,
+            "alarmText" : "PERFORMANCE TEST ALARM 6",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1007,
+            "alarmText" : "PERFORMANCE TEST ALARM 7",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1008,
+            "alarmText" : "PERFORMANCE TEST ALARM 8",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 3,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1009,
+            "alarmText" : "PERFORMANCE TEST ALARM 9",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1010,
+            "alarmText" : "PERFORMANCE TEST ALARM 10",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1011,
+            "alarmText" : "PERFORMANCE TEST ALARM 11",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1012,
+            "alarmText" : "PERFORMANCE TEST ALARM 12",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 1
+        },
+        {
+            "alarmId" : 1013,
+            "alarmText" : "PERFORMANCE TEST ALARM 13",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1014,
+            "alarmText" : "PERFORMANCE TEST ALARM 14",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1015,
+            "alarmText" : "PERFORMANCE TEST ALARM 15",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1016,
+            "alarmText" : "PERFORMANCE TEST ALARM 16",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 2
+        },
+        {
+            "alarmId" : 1017,
+            "alarmText" : "PERFORMANCE TEST ALARM 17",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1018,
+            "alarmText" : "PERFORMANCE TEST ALARM 18",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1019,
+            "alarmText" : "PERFORMANCE TEST ALARM 19",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 3
+        },
+        {
+            "alarmId" : 1020,
+            "alarmText" : "PERFORMANCE TEST ALARM 20",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1021,
+            "alarmText" : "PERFORMANCE TEST ALARM 21",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1022,
+            "alarmText" : "PERFORMANCE TEST ALARM 22",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1023,
+            "alarmText" : "PERFORMANCE TEST ALARM 23",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 1,
+            "clearDelay" : 1
+        },
+        {
+            "alarmId" : 1024,
+            "alarmText" : "PERFORMANCE TEST ALARM 24",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1025,
+            "alarmText" : "PERFORMANCE TEST ALARM 25",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1026,
+            "alarmText" : "PERFORMANCE TEST ALARM 26",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 2,
+            "clearDelay" : 1
+        },
+        {
+            "alarmId" : 1027,
+            "alarmText" : "PERFORMANCE TEST ALARM 27",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1028,
+            "alarmText" : "PERFORMANCE TEST ALARM 28",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1029,
+            "alarmText" : "PERFORMANCE TEST ALARM 29",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 3,
+            "clearDelay" : 1
+        },
+        {
+            "alarmId" : 1030,
+            "alarmText" : "PERFORMANCE TEST ALARM 30",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1031,
+            "alarmText" : "PERFORMANCE TEST ALARM 31",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1032,
+            "alarmText" : "PERFORMANCE TEST ALARM 32",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1033,
+            "alarmText" : "PERFORMANCE TEST ALARM 33",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 1,
+            "clearDelay" : 1
+        },
+        {
+            "alarmId" : 1034,
+            "alarmText" : "PERFORMANCE TEST ALARM 34",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1035,
+            "alarmText" : "PERFORMANCE TEST ALARM 35",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1036,
+            "alarmText" : "PERFORMANCE TEST ALARM 36",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 1,
+            "clearDelay" : 2
+        },
+        {
+            "alarmId" : 1037,
+            "alarmText" : "PERFORMANCE TEST ALARM 37",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1038,
+            "alarmText" : "PERFORMANCE TEST ALARM 38",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1039,
+            "alarmText" : "PERFORMANCE TEST ALARM 39",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 1,
+            "clearDelay" : 3
+        },
+        {
+            "alarmId" : 1040,
+            "alarmText" : "PERFORMANCE TEST ALARM 40",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1041,
+            "alarmText" : "PERFORMANCE TEST ALARM 41",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1042,
+            "alarmText" : "PERFORMANCE TEST ALARM 42",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 1,
+            "clearDelay" : 1
+        },
+        {
+            "alarmId" : 1043,
+            "alarmText" : "PERFORMANCE TEST ALARM 43",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1044,
+            "alarmText" : "PERFORMANCE TEST ALARM 44",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1045,
+            "alarmText" : "PERFORMANCE TEST ALARM 45",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 2,
+            "clearDelay" : 2
+        },
+        {
+            "alarmId" : 1046,
+            "alarmText" : "PERFORMANCE TEST ALARM 46",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1047,
+            "alarmText" : "PERFORMANCE TEST ALARM 47",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1048,
+            "alarmText" : "PERFORMANCE TEST ALARM 48",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 3,
+            "clearDelay" : 3
+        },
+        {
+            "alarmId" : 1049,
+            "alarmText" : "PERFORMANCE TEST ALARM 49",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        },
+        {
+            "alarmId" : 1050,
+            "alarmText" : "PERFORMANCE TEST ALARM 50",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
+        }
+   ]
+}
index e12fb0f..ba7c683 100755 (executable)
             "alarmId" : 1001,
             "alarmText" : "PERFORMANCE TEST ALARM 1",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1002,
             "alarmText" : "PERFORMANCE TEST ALARM 2",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1003,
             "alarmText" : "PERFORMANCE TEST ALARM 3",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1004,
             "alarmText" : "PERFORMANCE TEST ALARM 4",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1005,
             "alarmText" : "PERFORMANCE TEST ALARM 5",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1006,
             "alarmText" : "PERFORMANCE TEST ALARM 6",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1007,
             "alarmText" : "PERFORMANCE TEST ALARM 7",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1008,
             "alarmText" : "PERFORMANCE TEST ALARM 8",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1009,
             "alarmText" : "PERFORMANCE TEST ALARM 9",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1010,
             "alarmText" : "PERFORMANCE TEST ALARM 10",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1011,
             "alarmText" : "PERFORMANCE TEST ALARM 11",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1012,
             "alarmText" : "PERFORMANCE TEST ALARM 12",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1013,
             "alarmText" : "PERFORMANCE TEST ALARM 13",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1014,
             "alarmText" : "PERFORMANCE TEST ALARM 14",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1015,
             "alarmText" : "PERFORMANCE TEST ALARM 15",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1016,
             "alarmText" : "PERFORMANCE TEST ALARM 16",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1017,
             "alarmText" : "PERFORMANCE TEST ALARM 17",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1018,
             "alarmText" : "PERFORMANCE TEST ALARM 18",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1019,
             "alarmText" : "PERFORMANCE TEST ALARM 19",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1020,
             "alarmText" : "PERFORMANCE TEST ALARM 20",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1021,
             "alarmText" : "PERFORMANCE TEST ALARM 21",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1022,
             "alarmText" : "PERFORMANCE TEST ALARM 22",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1023,
             "alarmText" : "PERFORMANCE TEST ALARM 23",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1024,
             "alarmText" : "PERFORMANCE TEST ALARM 24",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1025,
             "alarmText" : "PERFORMANCE TEST ALARM 25",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1026,
             "alarmText" : "PERFORMANCE TEST ALARM 26",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1027,
             "alarmText" : "PERFORMANCE TEST ALARM 27",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1028,
             "alarmText" : "PERFORMANCE TEST ALARM 28",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1029,
             "alarmText" : "PERFORMANCE TEST ALARM 29",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1030,
             "alarmText" : "PERFORMANCE TEST ALARM 30",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1031,
             "alarmText" : "PERFORMANCE TEST ALARM 31",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1032,
             "alarmText" : "PERFORMANCE TEST ALARM 32",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1033,
             "alarmText" : "PERFORMANCE TEST ALARM 33",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1034,
             "alarmText" : "PERFORMANCE TEST ALARM 34",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1035,
             "alarmText" : "PERFORMANCE TEST ALARM 35",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1036,
             "alarmText" : "PERFORMANCE TEST ALARM 36",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1037,
             "alarmText" : "PERFORMANCE TEST ALARM 37",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1038,
             "alarmText" : "PERFORMANCE TEST ALARM 38",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1039,
             "alarmText" : "PERFORMANCE TEST ALARM 39",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1040,
             "alarmText" : "PERFORMANCE TEST ALARM 40",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1041,
             "alarmText" : "PERFORMANCE TEST ALARM 41",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1042,
             "alarmText" : "PERFORMANCE TEST ALARM 42",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1043,
             "alarmText" : "PERFORMANCE TEST ALARM 43",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1044,
             "alarmText" : "PERFORMANCE TEST ALARM 44",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1045,
             "alarmText" : "PERFORMANCE TEST ALARM 45",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1046,
             "alarmText" : "PERFORMANCE TEST ALARM 46",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1047,
             "alarmText" : "PERFORMANCE TEST ALARM 47",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1048,
             "alarmText" : "PERFORMANCE TEST ALARM 48",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1049,
             "alarmText" : "PERFORMANCE TEST ALARM 49",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         },
         {
             "alarmId" : 1050,
             "alarmText" : "PERFORMANCE TEST ALARM 50",
             "eventType" : "Performance test",
-            "operationInstructions" : "Not defined"
+            "operationInstructions" : "Not defined",
+            "raiseDelay" : 0,
+            "clearDelay" : 0
         }
    ]
 }