From 18fd03c7dd490347eeec3ed60e176fb3a8ccc3c7 Mon Sep 17 00:00:00 2001 From: Anssi Mannila Date: Thu, 29 Oct 2020 10:01:00 +0200 Subject: [PATCH 1/1] LN0739_FM_FR9: Add configured delay for raise/clear alarm handling Change-Id: Iec0b53a825fdbca2221672910f517aaeaba2b44c Signed-off-by: Anssi Mannila --- alarm/types.go | 4 +- build/container-tag.yaml | 2 +- cli/alarm-cli.go | 42 ++- definitions/alarm-definition.json | 24 +- docs/user-guide.rst | 12 +- go.mod | 2 - manager/cmd/manager.go | 130 ++++++-- manager/cmd/manager_test.go | 144 +++++++++ manager/cmd/restapi.go | 2 + testresources/perf-alarm-definition-delay.json | 404 +++++++++++++++++++++++++ testresources/perf-alarm-definition.json | 200 +++++++++--- 11 files changed, 861 insertions(+), 105 deletions(-) create mode 100755 testresources/perf-alarm-definition-delay.json diff --git a/alarm/types.go b/alarm/types.go index 2aaa84b..200d435 100755 --- a/alarm/types.go +++ b/alarm/types.go @@ -151,7 +151,9 @@ type AlarmDefinition struct { AlarmId int `json:"alarmId"` AlarmText string `json:"alarmText"` EventType string `json:"eventType"` - OperationInstructions string `json:"operationinstructions"` + OperationInstructions string `json:"operationInstructions"` + RaiseDelay int `json:"raiseDelay"` + ClearDelay int `json:"clearDelay"` } var RICAlarmDefinitions map[int]*AlarmDefinition diff --git a/build/container-tag.yaml b/build/container-tag.yaml index 4f49c1f..599041f 100644 --- a/build/container-tag.yaml +++ b/build/container-tag.yaml @@ -2,4 +2,4 @@ # By default this file is in the docker build directory, # but the location can configured in the JJB template. --- -tag: 0.5.6 +tag: 0.5.7 diff --git a/cli/alarm-cli.go b/cli/alarm-cli.go index 9a18015..b213ec6 100755 --- a/cli/alarm-cli.go +++ b/cli/alarm-cli.go @@ -142,6 +142,8 @@ func main() { AddFlag("atx", "alarm text", commando.String, nil). AddFlag("ety", "event type", commando.String, nil). AddFlag("oin", "operation instructions", commando.String, nil). + AddFlag("rad", "Raise alarm delay", commando.Int, 0). + AddFlag("cad", "Clear alarm delay", commando.Int, 0). AddFlag("host", "Alarm manager host address", commando.String, alarmManagerHost). AddFlag("port", "Alarm manager host address", commando.String, "8080"). SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) { @@ -292,15 +294,18 @@ func displayAlarms(alarms []AlarmNotification, isHistory bool) { } for _, a := range alarms { + // Do not show alarm before raiseDelay has elapsed alarmTime := time.Unix(0, a.AlarmTime).Format("02/01/2006, 15:04:05") if isHistory { t.AppendRows([]table.Row{ {a.AlarmId, a.SpecificProblem, a.ManagedObjectId, a.ApplicationId, a.IdentifyingInfo, a.PerceivedSeverity, a.AdditionalInfo, a.AlarmAction, alarmTime}, }) } else { - t.AppendRows([]table.Row{ - {a.AlarmId, a.SpecificProblem, a.ManagedObjectId, a.ApplicationId, a.IdentifyingInfo, a.PerceivedSeverity, a.AdditionalInfo, alarmTime}, - }) + if a.AlarmDefinition.RaiseDelay == 0 { + t.AppendRows([]table.Row{ + {a.AlarmId, a.SpecificProblem, a.ManagedObjectId, a.ApplicationId, a.IdentifyingInfo, a.PerceivedSeverity, a.AdditionalInfo, alarmTime}, + }) + } } } @@ -336,6 +341,9 @@ func postAlarmDefinition(flags map[string]commando.FlagValue) { alarmtxt, _ := flags["atx"].GetString() etype, _ := flags["ety"].GetString() operation, _ := flags["oin"].GetString() + raiseDelay, _ := flags["rad"].GetInt() + clearDelay, _ := flags["cad"].GetInt() + targetUrl := fmt.Sprintf("http://%s:%s/ric/v1/alarms/define", host, port) var alarmdefinition alarm.AlarmDefinition @@ -343,6 +351,8 @@ func postAlarmDefinition(flags map[string]commando.FlagValue) { alarmdefinition.AlarmText = alarmtxt alarmdefinition.EventType = etype alarmdefinition.OperationInstructions = operation + alarmdefinition.RaiseDelay = raiseDelay + alarmdefinition.ClearDelay = clearDelay m := CliAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarmdefinition}} jsonData, err := json.Marshal(m) @@ -426,7 +436,6 @@ func conductperformancetest(flags map[string]commando.FlagValue) { } else { fmt.Println("reading performance alarm definitions from json file failed ") } - } func peakPerformanceTest(flags map[string]commando.FlagValue) { @@ -466,7 +475,10 @@ func enduranceTest(flags map[string]commando.FlagValue) { } func readPerfAlarmObjectFromJson() error { + filename := os.Getenv("PERF_OBJ_FILE") + fmt.Printf("readPerfAlarmObjectFromJson: filename = %s\n", filename) + file, err := ioutil.ReadFile(filename) if err == nil { data := RicPerfAlarmObjects{} @@ -483,18 +495,22 @@ func readPerfAlarmObjectFromJson() error { CLIPerfAlarmObjects[alarmObject.SpecificProblem] = ricAlarmObject } } else { - fmt.Println("readPerfAlarmObjectFromJson: json.Unmarshal failed with error ", err) + fmt.Println("readPerfAlarmObjectFromJson: json.Unmarshal failed with error: ", err) return err } } else { - fmt.Println("readPerfAlarmObjectFromJson: ioutil.ReadFile failed with error ", err) + fmt.Printf("readPerfAlarmObjectFromJson: ioutil.ReadFile failed with error: %v, filename: %s\n", err, filename) + fmt.Printf("readPerfAlarmObjectFromJson: current directory: %s\n", getCurrentDirectory()) return err } return nil } func readPerfAlarmDefinitionFromJson() error { + filename := os.Getenv("PERF_DEF_FILE") + fmt.Printf("ReadPerfAlarmDefinitionFromJson: filename = %s\n", filename) + file, err := ioutil.ReadFile(filename) if err == nil { data := CliAlarmDefinitions{} @@ -511,6 +527,8 @@ func readPerfAlarmDefinitionFromJson() error { ricAlarmDefintion.AlarmText = alarmDefinition.AlarmText ricAlarmDefintion.EventType = alarmDefinition.EventType ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions + ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay + ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay CliPerfAlarmDefinitions.AlarmDefinitions = append(CliPerfAlarmDefinitions.AlarmDefinitions, ricAlarmDefintion) } } @@ -519,7 +537,9 @@ func readPerfAlarmDefinitionFromJson() error { return err } } else { - fmt.Println("ReadPerfAlarmDefinitionFromJson: ioutil.ReadFile failed with error: ", err) + fmt.Printf("ReadPerfAlarmDefinitionFromJson: ioutil.ReadFile failed with error: %v, filename: %s\n", err, filename) + fmt.Printf("ReadPerfAlarmDefinitionFromJson: current directory: %s\n", getCurrentDirectory()) + return err } return nil @@ -681,3 +701,11 @@ func newAlertManagerClient(amAddress string, amBaseUrl string, amSchemes []strin cr := clientruntime.New(amAddress, amBaseUrl, amSchemes) return client.New(cr, strfmt.Default) } + +func getCurrentDirectory() string { + dir, err := os.Getwd() + if err != nil { + fmt.Println(err) + } + return dir +} \ No newline at end of file diff --git a/definitions/alarm-definition.json b/definitions/alarm-definition.json index eebf842..ce32693 100755 --- a/definitions/alarm-definition.json +++ b/definitions/alarm-definition.json @@ -4,37 +4,49 @@ "alarmId" : 8004, "alarmText" : "RIC ROUTING TABLE DISTRIBUTION FAILED", "eventType" : "processingError", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 8005, "alarmText" : "TCP CONNECTIVITY LOST TO DBAAS", "eventType" : "communication", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 8006, "alarmText" : "E2 CONNECTIVITY LOST TO G-NODEB", "eventType" : "communication", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 8007, "alarmText" : "E2 CONNECTIVITY LOST TO E-NODEB", "eventType" : "communication", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 8008, "alarmText" : "ACTIVE ALARM EXCEED MAX THRESHOLD", "eventType" : "equipment", - "operationInstructions" : "Clear alarms or raise threshold" + "operationInstructions" : "Clear alarms or raise threshold", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 8009, "alarmText" : "ALARM HISTORY EXCEED MAX THRESHOLD", "eventType" : "equipment", - "operationInstructions" : "Clear alarms or raise threshold" + "operationInstructions" : "Clear alarms or raise threshold", + "raiseDelay" : 0, + "clearDelay" : 0 } ] } diff --git a/docs/user-guide.rst b/docs/user-guide.rst index e531f7c..bc08e13 100755 --- a/docs/user-guide.rst +++ b/docs/user-guide.rst @@ -147,6 +147,8 @@ CLI commands can have some of the following parameters --atx Alarm text string, example string: E2 CONNECTIVITY LOST TO E-NODEB --ety Event type string, example string: Communication error --oin Operation instructions string, example string: Not defined + --rad Raise alarm delay in seconds. Default value = 0 + --cad Clear alarm delay in seconds. Default value = 0 --prf Performance profile id, possible values: 1 = peak performance test or 2 = endurance test --nal Number of alarms, example value: 50 --aps Alarms per second, example value: 1 @@ -226,11 +228,11 @@ CLI command examples: .. code-block:: none - Syntax: cli/alarm-cli define --aid 8007 --atx "E2 CONNECTIVITY LOST TO E-NODEB" --ety "Communication error" --oin "Not defined" [--host] [--port] + Syntax: cli/alarm-cli define --aid 8007 --atx "E2 CONNECTIVITY LOST TO E-NODEB" --ety "Communication error" --oin "Not defined" [--rad] [--cad] [--host] [--port] - Example: cli/alarm-cli define --aid 8007 --atx "E2 CONNECTIVITY LOST TO E-NODEB" --ety "Communication error" --oin "Not defined" + Example: cli/alarm-cli define --aid 8007 --atx "E2 CONNECTIVITY LOST TO E-NODEB" --ety "Communication error" --oin "Not defined" --rad 0 --cad 0 - Example: cli/alarm-cli define --aid 8007 --atx "E2 CONNECTIVITY LOST TO E-NODEB" --ety "Communication error" --oin "Not defined" --host localhost --port 8080 + Example: cli/alarm-cli define --aid 8007 --atx "E2 CONNECTIVITY LOST TO E-NODEB" --ety "Communication error" --oin "Not defined" --rad 0 --cad 0 --host localhost --port 8080 Delete existing alarm definition: @@ -319,11 +321,11 @@ Below are examples for REST interface. Curl tool is used to send REST commands. Add one new alarm definition: - Example: curl -X POST "http://localhost:8080/ric/v1/alarms/define" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"alarmdefinitions\": [{\"alarmId\": 8007, \"alarmText\": \"E2 CONNECTIVITY LOST TO E-NODEB\", \"eventtype\": \"Communication error\", \"operationinstructions\": \"Not defined\"}]}" + Example: curl -X POST "http://localhost:8080/ric/v1/alarms/define" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"alarmdefinitions\": [{\"alarmId\": 8007, \"alarmText\": \"E2 CONNECTIVITY LOST TO E-NODEB\", \"eventtype\": \"Communication error\", \"operationinstructions\": \"Not defined\, \"raiseDelay\": 1, \"clearDelay\": 1"}]}" Add two new alarm definitions: - Example: curl -X POST "http://localhost:8080/ric/v1/alarms/define" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"alarmdefinitions\": [{\"alarmId\": 8007, \"alarmText\": \"E2 CONNECTIVITY LOST TO E-NODEB\", \"eventtype\": \"Communication error\", \"operationinstructions\": \"Not defined\"},{\"alarmId\": 8008, \"alarmText\": \"ACTIVE ALARM EXCEED MAX THRESHOLD\", \"eventtype\": \"storage warning\", \"operationinstructions\": \"Clear alarms or raise threshold\"}]}" + Example: curl -X POST "http://localhost:8080/ric/v1/alarms/define" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"alarmdefinitions\": [{\"alarmId\": 8007, \"alarmText\": \"E2 CONNECTIVITY LOST TO E-NODEB\", \"eventtype\": \"Communication error\", \"operationinstructions\": \"Not defined\, \"raiseDelay\": 0, \"clearDelay\": 0"},{\"alarmId\": 8008, \"alarmText\": \"ACTIVE ALARM EXCEED MAX THRESHOLD\", \"eventtype\": \"storage warning\", \"operationinstructions\": \"Clear alarms or raise threshold\", \"raiseDelay\": 0, \"clearDelay\": 0}]}" Delete one existing alarm definition: diff --git a/go.mod b/go.mod index 0fb43fa..b6ae937 100644 --- a/go.mod +++ b/go.mod @@ -20,8 +20,6 @@ require ( github.com/gorilla/mux v1.7.1 github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect github.com/prometheus/alertmanager v0.20.0 github.com/spf13/viper v1.6.2 github.com/stretchr/testify v1.5.1 diff --git a/manager/cmd/manager.go b/manager/cmd/manager.go index 2b5af17..f7ad81e 100755 --- a/manager/cmd/manager.go +++ b/manager/cmd/manager.go @@ -24,6 +24,11 @@ import ( "bytes" "encoding/json" "fmt" + "io/ioutil" + "net/http" + "os" + "time" + "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm" app "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp" clientruntime "github.com/go-openapi/runtime/client" @@ -32,10 +37,6 @@ import ( "github.com/prometheus/alertmanager/api/v2/client/alert" "github.com/prometheus/alertmanager/api/v2/models" "github.com/spf13/viper" - "io/ioutil" - "net/http" - "os" - "time" ) func (a *AlarmManager) StartAlertTimer() { @@ -78,15 +79,16 @@ func (a *AlarmManager) HandleAlarms(rp *app.RMRParams) (*alert.PostAlertsOK, err func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) { a.mutex.Lock() - - if _, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok { + alarmDef := &alarm.AlarmDefinition{} + var ok bool + if alarmDef, ok = alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok { app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem) a.mutex.Unlock() return nil, nil } - // Suppress duplicate alarms idx, found := a.IsMatchFound(m.Alarm) + // Suppress duplicate alarms if found && m.AlarmAction == alarm.AlarmActionRaise { app.Logger.Info("Duplicate alarm found, suppressing ...") if m.PerceivedSeverity == a.activeAlarms[idx].PerceivedSeverity { @@ -102,24 +104,9 @@ func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK, // Clear alarm if found from active alarm list if m.AlarmAction == alarm.AlarmActionClear { if found { - a.UpdateAlarmFields(a.activeAlarms[idx].AlarmId, m) - a.alarmHistory = append(a.alarmHistory, *m) - a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active") - if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) { - app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold") - a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history") - } - - if a.exceededActiveAlarmOn && m.Alarm.SpecificProblem == alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD { - a.exceededActiveAlarmOn = false - } - - if a.exceededAlarmHistoryOn && m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD { - a.exceededAlarmHistoryOn = false + if a.ProcessClearAlarm(m, alarmDef, idx) == false { + return nil, nil } - - a.WriteAlarmInfoToPersistentVolume() - if a.postClear { a.mutex.Unlock() @@ -138,11 +125,9 @@ func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK, // New alarm -> update active alarms and post to Alert Manager if m.AlarmAction == alarm.AlarmActionRaise { - a.UpdateAlarmFields(a.GenerateAlarmId(), m) - a.UpdateAlarmLists(m) - a.WriteAlarmInfoToPersistentVolume() - a.mutex.Unlock() - + if a.ProcessRaiseAlarm(m, alarmDef) == false { + return nil, nil + } // Send alarm notification to NOMA, if enabled if app.Config.GetBool("controls.noma.enabled") { return a.PostAlarm(m) @@ -154,6 +139,75 @@ func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK, return nil, nil } +func (a *AlarmManager)ProcessRaiseAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition) bool { + app.Logger.Debug("Raise alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m) + // RaiseDelay > 0 in an alarm object in active alarm table indicates that raise delay is still ongoing for the alarm + m.AlarmDefinition.RaiseDelay = alarmDef.RaiseDelay + a.UpdateAlarmFields(a.GenerateAlarmId(), m) + a.UpdateActiveAlarmList(m) + a.mutex.Unlock() + if alarmDef.RaiseDelay > 0 { + timerDelay(alarmDef.RaiseDelay) + a.mutex.Lock() + // Alarm may have been deleted from active alarms table during delay or table index may have changed + idx, found := a.IsMatchFound(m.Alarm) + if found { + // Alarm is not showed in active alarms or alarm history via CLI before RaiseDelay has elapsed, i.e the value is 0 + a.activeAlarms[idx].AlarmDefinition.RaiseDelay = 0 + app.Logger.Debug("Raise after delay alarmDef.RaiseDelay = %v, AlarmNotification = %v", alarmDef.RaiseDelay, *m) + a.mutex.Unlock() + } else { + app.Logger.Debug("Alarm deleted during raise delay. AlarmNotification = %v", *m) + a.mutex.Unlock() + return false + } + } + m.AlarmDefinition.RaiseDelay = 0 + a.UpdateAlarmHistoryList(m) + a.WriteAlarmInfoToPersistentVolume() + return true +} + +func (a *AlarmManager)ProcessClearAlarm(m *AlarmNotification, alarmDef *alarm.AlarmDefinition, idx int) bool { + app.Logger.Debug("Clear alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m) + if alarmDef.ClearDelay > 0 { + a.mutex.Unlock() + timerDelay(alarmDef.ClearDelay) + app.Logger.Debug("Clear after delay alarmDef.ClearDelay = %v, AlarmNotification = %v", alarmDef.ClearDelay, *m) + a.mutex.Lock() + // Another alarm clear may have happened during delay and active alarms table index changed + var found bool + idx, found = a.IsMatchFound(m.Alarm) + if !found { + app.Logger.Debug("Alarm not anymore in the active alarms table. AlarmNotification = %v", *m) + a.mutex.Unlock() + return false + } + } + a.UpdateAlarmFields(a.activeAlarms[idx].AlarmId, m) + a.alarmHistory = append(a.alarmHistory, *m) + a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active") + if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) { + app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold") + a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history") + } + + if a.exceededActiveAlarmOn && m.Alarm.SpecificProblem == alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD { + a.exceededActiveAlarmOn = false + } + + if a.exceededAlarmHistoryOn && m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD { + a.exceededAlarmHistoryOn = false + } + a.WriteAlarmInfoToPersistentVolume() + return true +} + +func timerDelay(delay int) { + timer := time.NewTimer(time.Duration(delay) * time.Second) + <-timer.C +} + func (a *AlarmManager) IsMatchFound(newAlarm alarm.Alarm) (int, bool) { for i, m := range a.activeAlarms { if m.ManagedObjectId == newAlarm.ManagedObjectId && m.ApplicationId == newAlarm.ApplicationId && @@ -198,21 +252,29 @@ func (a *AlarmManager) GenerateThresholdAlarm(sp int, data string) bool { return true } -func (a *AlarmManager) UpdateAlarmLists(newAlarm *AlarmNotification) { +func (a *AlarmManager) UpdateActiveAlarmList(newAlarm *AlarmNotification) { /* If maximum number of active alarms is reached, an error log writing is made, and new alarm indicating the problem is raised. - The attempt to raise the alarm next time will be supressed when found as duplicate. */ + The attempt to raise the alarm next time will be suppressed when found as duplicate. */ if (len(a.activeAlarms) >= a.maxActiveAlarms) && (a.exceededActiveAlarmOn == false) { app.Logger.Warn("active alarm count exceeded maxActiveAlarms threshold") a.exceededActiveAlarmOn = a.GenerateThresholdAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, "active") } + // @todo: For now just keep the active alarms in-memory. Use SDL later for persistence + a.activeAlarms = append(a.activeAlarms, *newAlarm) +} + +func (a *AlarmManager) UpdateAlarmHistoryList(newAlarm *AlarmNotification) { + /* If maximum number of events in alarm history is reached, an error log writing is made, + and new alarm indicating the problem is raised. The attempt to add new event time will + be suppressed */ + if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) { app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold") a.exceededAlarmHistoryOn = a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history") } - // @todo: For now just keep the alarms (both active and history) in-memory. Use SDL later for persistence - a.activeAlarms = append(a.activeAlarms, *newAlarm) + // @todo: For now just keep the alarms history in-memory. Use SDL later for persistence a.alarmHistory = append(a.alarmHistory, *newAlarm) } @@ -322,6 +384,8 @@ func (a *AlarmManager) ReadAlarmDefinitionFromJson() { ricAlarmDefintion.AlarmText = alarmDefinition.AlarmText ricAlarmDefintion.EventType = alarmDefinition.EventType ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions + ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay + ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion } } diff --git a/manager/cmd/manager_test.go b/manager/cmd/manager_test.go index 2589a91..b75cae5 100755 --- a/manager/cmd/manager_test.go +++ b/manager/cmd/manager_test.go @@ -89,36 +89,48 @@ func TestSetAlarmDefinitions(t *testing.T) { alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED" alarm8004Definition.EventType = "Processing error" alarm8004Definition.OperationInstructions = "Not defined" + alarm8004Definition.RaiseDelay = 0 + alarm8004Definition.ClearDelay = 0 var alarm8005Definition alarm.AlarmDefinition alarm8005Definition.AlarmId = alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS alarm8005Definition.AlarmText = "TCP CONNECTIVITY LOST TO DBAAS" alarm8005Definition.EventType = "Communication error" alarm8005Definition.OperationInstructions = "Not defined" + alarm8005Definition.RaiseDelay = 0 + alarm8005Definition.ClearDelay = 0 var alarm8006Definition alarm.AlarmDefinition alarm8006Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_GNODEB alarm8006Definition.AlarmText = "E2 CONNECTIVITY LOST TO G-NODEB" alarm8006Definition.EventType = "Communication error" alarm8006Definition.OperationInstructions = "Not defined" + alarm8006Definition.RaiseDelay = 0 + alarm8006Definition.ClearDelay = 0 var alarm8007Definition alarm.AlarmDefinition alarm8007Definition.AlarmId = alarm.E2_CONNECTIVITY_LOST_TO_ENODEB alarm8007Definition.AlarmText = "E2 CONNECTIVITY LOST TO E-NODEB" alarm8007Definition.EventType = "Communication error" alarm8007Definition.OperationInstructions = "Not defined" + alarm8007Definition.RaiseDelay = 0 + alarm8007Definition.ClearDelay = 0 var alarm8008Definition alarm.AlarmDefinition alarm8008Definition.AlarmId = alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD alarm8008Definition.AlarmText = "ACTIVE ALARM EXCEED MAX THRESHOLD" alarm8008Definition.EventType = "storage warning" alarm8008Definition.OperationInstructions = "Clear alarms or raise threshold" + alarm8008Definition.RaiseDelay = 0 + alarm8008Definition.ClearDelay = 0 var alarm8009Definition alarm.AlarmDefinition alarm8009Definition.AlarmId = alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD alarm8009Definition.AlarmText = "ALARM HISTORY EXCEED MAX THRESHOLD" alarm8009Definition.EventType = "storage warning" alarm8009Definition.OperationInstructions = "Clear alarms or raise threshold" + alarm8009Definition.RaiseDelay = 0 + alarm8009Definition.ClearDelay = 0 pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition, &alarm8005Definition, &alarm8006Definition, &alarm8007Definition, &alarm8008Definition, &alarm8009Definition}} pbodyEn, _ := json.Marshal(pbodyParams) @@ -182,6 +194,8 @@ func TestDeleteAlarmDefinitions(t *testing.T) { alarm8004Definition.AlarmText = "RIC ROUTING TABLE DISTRIBUTION FAILED" alarm8004Definition.EventType = "Processing error" alarm8004Definition.OperationInstructions = "Not defined" + alarm8004Definition.RaiseDelay = 0 + alarm8004Definition.ClearDelay = 0 pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm8004Definition}} pbodyEn, _ := json.Marshal(pbodyParams) req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn)) @@ -346,6 +360,136 @@ func TestGetPrometheusAlerts(t *testing.T) { ts.Close() } +func TestDelayedAlarmRaiseAndClear(t *testing.T) { + xapp.Logger.Info("TestDelayedAlarmRaiseAndClear") + + activeAlarmsBeforeTest := len(alarmManager.activeAlarms) + alarmHistoryBeforeTest := len(alarmManager.alarmHistory) + + // Add new alarm definition + var alarm9999Definition alarm.AlarmDefinition + alarm9999Definition.AlarmId = 9999 + alarm9999Definition.AlarmText = "DELAYED TEST ALARM" + alarm9999Definition.EventType = "Test type" + alarm9999Definition.OperationInstructions = "Not defined" + alarm9999Definition.RaiseDelay = 1 + alarm9999Definition.ClearDelay = 1 + pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}} + pbodyEn, _ := json.Marshal(pbodyParams) + req, _ := http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn)) + handleFunc := http.HandlerFunc(alarmManager.SetAlarmDefinition) + response := executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + // Verify 9999 alarm definition + req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil) + vars := map[string]string{"alarmId": strconv.FormatUint(8004, 10)} + req = mux.SetURLVars(req, vars) + handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition) + response = executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{}) + defer ts.Close() + + // Raise alarm. Posting alert and updating alarm history should be delayed + a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1") + assert.Nil(t, alarmer.Raise(a), "raise failed") + VerifyAlarm(t, a, activeAlarmsBeforeTest + 1) + + // Clear the alarm and check the alarm is removed. Posting alert clear and updating alarm history should be delayed + assert.Nil(t, alarmer.Clear(a), "clear failed") + + time.Sleep(time.Duration(2) * time.Second) + assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest) + assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest + 2) +} + +func TestDelayedAlarmRaiseAndClear2(t *testing.T) { + xapp.Logger.Info("TestDelayedAlarmRaiseAndClear2") + + activeAlarmsBeforeTest := len(alarmManager.activeAlarms) + alarmHistoryBeforeTest := len(alarmManager.alarmHistory) + + ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{}) + defer ts.Close() + + // Raise two alarms. The first should be delayed + a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1") + assert.Nil(t, alarmer.Raise(a), "raise failed") + VerifyAlarm(t, a, activeAlarmsBeforeTest + 1) + + b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1") + assert.Nil(t, alarmer.Raise(b), "raise failed") + VerifyAlarm(t, b, activeAlarmsBeforeTest + 2) + + // Clear two alarms. The first should be delayed. Check the alarms are removed + assert.Nil(t, alarmer.Clear(a), "clear failed") + assert.Nil(t, alarmer.Clear(b), "clear failed") + + time.Sleep(time.Duration(2) * time.Second) + assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest) + assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest + 4) +} + +func TestDelayedAlarmRaiseAndClear3(t *testing.T) { + xapp.Logger.Info("TestDelayedAlarmRaiseAndClear3") + + // Delete exisitng 9999 alarm definition + req, _ := http.NewRequest("DELETE", "/ric/v1/alarms/define", nil) + vars := map[string]string{"alarmId": strconv.FormatUint(9999, 10)} + req = mux.SetURLVars(req, vars) + handleFunc := http.HandlerFunc(alarmManager.DeleteAlarmDefinition) + response := executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + // Add updated 9999 alarm definition + var alarm9999Definition alarm.AlarmDefinition + alarm9999Definition.AlarmId = 9999 + alarm9999Definition.AlarmText = "DELAYED TEST ALARM" + alarm9999Definition.EventType = "Test type" + alarm9999Definition.OperationInstructions = "Not defined" + alarm9999Definition.RaiseDelay = 1 + alarm9999Definition.ClearDelay = 0 + pbodyParams := RicAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarm9999Definition}} + pbodyEn, _ := json.Marshal(pbodyParams) + req, _ = http.NewRequest("POST", "/ric/v1/alarms/define", bytes.NewBuffer(pbodyEn)) + handleFunc = http.HandlerFunc(alarmManager.SetAlarmDefinition) + response = executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + // Verify 9999 alarm definition + req, _ = http.NewRequest("GET", "/ric/v1/alarms/define", nil) + vars = map[string]string{"alarmId": strconv.FormatUint(8004, 10)} + req = mux.SetURLVars(req, vars) + handleFunc = http.HandlerFunc(alarmManager.GetAlarmDefinition) + response = executeRequest(req, handleFunc) + checkResponseCode(t, http.StatusOK, response.Code) + + activeAlarmsBeforeTest := len(alarmManager.activeAlarms) + alarmHistoryBeforeTest := len(alarmManager.alarmHistory) + + ts := CreatePromAlertSimulator(t, "POST", "/api/v2/alerts", http.StatusOK, models.LabelSet{}) + defer ts.Close() + + // Raise two alarms. The first should be delayed + a := alarmer.NewAlarm(9999, alarm.SeverityCritical, "Some App data", "eth 0 1") + assert.Nil(t, alarmer.Raise(a), "raise failed") + VerifyAlarm(t, a, activeAlarmsBeforeTest + 1) + + b := alarmer.NewAlarm(alarm.RIC_RT_DISTRIBUTION_FAILED, alarm.SeverityMajor, "Some App data", "eth 0 1") + assert.Nil(t, alarmer.Raise(b), "raise failed") + VerifyAlarm(t, b, activeAlarmsBeforeTest + 2) + + // Clear two alarms. The first should be delayed. Check the alarms are removed + assert.Nil(t, alarmer.Clear(a), "clear failed") + assert.Nil(t, alarmer.Clear(b), "clear failed") + + time.Sleep(time.Duration(2) * time.Second) + assert.Equal(t, len(alarmManager.activeAlarms), activeAlarmsBeforeTest) + assert.Equal(t, len(alarmManager.alarmHistory), alarmHistoryBeforeTest + 4) +} + func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string { receivedAlert := waitForEvent() diff --git a/manager/cmd/restapi.go b/manager/cmd/restapi.go index 412c71c..b8946df 100755 --- a/manager/cmd/restapi.go +++ b/manager/cmd/restapi.go @@ -95,6 +95,8 @@ func (a *AlarmManager) SetAlarmDefinition(w http.ResponseWriter, r *http.Request ricAlarmDefintion.AlarmText = alarmDefinition.AlarmText ricAlarmDefintion.EventType = alarmDefinition.EventType ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions + ricAlarmDefintion.RaiseDelay = alarmDefinition.RaiseDelay + ricAlarmDefintion.ClearDelay = alarmDefinition.ClearDelay alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion app.Logger.Debug("POST - alarm definition added for alarm id %v", alarmDefinition.AlarmId) } diff --git a/testresources/perf-alarm-definition-delay.json b/testresources/perf-alarm-definition-delay.json new file mode 100755 index 0000000..e766d74 --- /dev/null +++ b/testresources/perf-alarm-definition-delay.json @@ -0,0 +1,404 @@ +{ + "alarmdefinitions" : [ + { + "alarmId" : 1001, + "alarmText" : "PERFORMANCE TEST ALARM 1", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 1, + "clearDelay" : 0 + }, + { + "alarmId" : 1002, + "alarmText" : "PERFORMANCE TEST ALARM 2", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1003, + "alarmText" : "PERFORMANCE TEST ALARM 3", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1004, + "alarmText" : "PERFORMANCE TEST ALARM 4", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 2, + "clearDelay" : 0 + }, + { + "alarmId" : 1005, + "alarmText" : "PERFORMANCE TEST ALARM 5", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1006, + "alarmText" : "PERFORMANCE TEST ALARM 6", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1007, + "alarmText" : "PERFORMANCE TEST ALARM 7", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1008, + "alarmText" : "PERFORMANCE TEST ALARM 8", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 3, + "clearDelay" : 0 + }, + { + "alarmId" : 1009, + "alarmText" : "PERFORMANCE TEST ALARM 9", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1010, + "alarmText" : "PERFORMANCE TEST ALARM 10", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1011, + "alarmText" : "PERFORMANCE TEST ALARM 11", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1012, + "alarmText" : "PERFORMANCE TEST ALARM 12", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 1 + }, + { + "alarmId" : 1013, + "alarmText" : "PERFORMANCE TEST ALARM 13", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1014, + "alarmText" : "PERFORMANCE TEST ALARM 14", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1015, + "alarmText" : "PERFORMANCE TEST ALARM 15", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1016, + "alarmText" : "PERFORMANCE TEST ALARM 16", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 2 + }, + { + "alarmId" : 1017, + "alarmText" : "PERFORMANCE TEST ALARM 17", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1018, + "alarmText" : "PERFORMANCE TEST ALARM 18", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1019, + "alarmText" : "PERFORMANCE TEST ALARM 19", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 3 + }, + { + "alarmId" : 1020, + "alarmText" : "PERFORMANCE TEST ALARM 20", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1021, + "alarmText" : "PERFORMANCE TEST ALARM 21", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1022, + "alarmText" : "PERFORMANCE TEST ALARM 22", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1023, + "alarmText" : "PERFORMANCE TEST ALARM 23", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 1, + "clearDelay" : 1 + }, + { + "alarmId" : 1024, + "alarmText" : "PERFORMANCE TEST ALARM 24", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1025, + "alarmText" : "PERFORMANCE TEST ALARM 25", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1026, + "alarmText" : "PERFORMANCE TEST ALARM 26", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 2, + "clearDelay" : 1 + }, + { + "alarmId" : 1027, + "alarmText" : "PERFORMANCE TEST ALARM 27", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1028, + "alarmText" : "PERFORMANCE TEST ALARM 28", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1029, + "alarmText" : "PERFORMANCE TEST ALARM 29", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 3, + "clearDelay" : 1 + }, + { + "alarmId" : 1030, + "alarmText" : "PERFORMANCE TEST ALARM 30", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1031, + "alarmText" : "PERFORMANCE TEST ALARM 31", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1032, + "alarmText" : "PERFORMANCE TEST ALARM 32", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1033, + "alarmText" : "PERFORMANCE TEST ALARM 33", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 1, + "clearDelay" : 1 + }, + { + "alarmId" : 1034, + "alarmText" : "PERFORMANCE TEST ALARM 34", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1035, + "alarmText" : "PERFORMANCE TEST ALARM 35", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1036, + "alarmText" : "PERFORMANCE TEST ALARM 36", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 1, + "clearDelay" : 2 + }, + { + "alarmId" : 1037, + "alarmText" : "PERFORMANCE TEST ALARM 37", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1038, + "alarmText" : "PERFORMANCE TEST ALARM 38", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1039, + "alarmText" : "PERFORMANCE TEST ALARM 39", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 1, + "clearDelay" : 3 + }, + { + "alarmId" : 1040, + "alarmText" : "PERFORMANCE TEST ALARM 40", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1041, + "alarmText" : "PERFORMANCE TEST ALARM 41", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1042, + "alarmText" : "PERFORMANCE TEST ALARM 42", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 1, + "clearDelay" : 1 + }, + { + "alarmId" : 1043, + "alarmText" : "PERFORMANCE TEST ALARM 43", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1044, + "alarmText" : "PERFORMANCE TEST ALARM 44", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1045, + "alarmText" : "PERFORMANCE TEST ALARM 45", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 2, + "clearDelay" : 2 + }, + { + "alarmId" : 1046, + "alarmText" : "PERFORMANCE TEST ALARM 46", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1047, + "alarmText" : "PERFORMANCE TEST ALARM 47", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1048, + "alarmText" : "PERFORMANCE TEST ALARM 48", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 3, + "clearDelay" : 3 + }, + { + "alarmId" : 1049, + "alarmText" : "PERFORMANCE TEST ALARM 49", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + }, + { + "alarmId" : 1050, + "alarmText" : "PERFORMANCE TEST ALARM 50", + "eventType" : "Performance test", + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 + } + ] +} diff --git a/testresources/perf-alarm-definition.json b/testresources/perf-alarm-definition.json index e12fb0f..ba7c683 100755 --- a/testresources/perf-alarm-definition.json +++ b/testresources/perf-alarm-definition.json @@ -4,301 +4,401 @@ "alarmId" : 1001, "alarmText" : "PERFORMANCE TEST ALARM 1", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1002, "alarmText" : "PERFORMANCE TEST ALARM 2", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1003, "alarmText" : "PERFORMANCE TEST ALARM 3", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1004, "alarmText" : "PERFORMANCE TEST ALARM 4", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1005, "alarmText" : "PERFORMANCE TEST ALARM 5", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1006, "alarmText" : "PERFORMANCE TEST ALARM 6", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1007, "alarmText" : "PERFORMANCE TEST ALARM 7", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1008, "alarmText" : "PERFORMANCE TEST ALARM 8", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1009, "alarmText" : "PERFORMANCE TEST ALARM 9", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1010, "alarmText" : "PERFORMANCE TEST ALARM 10", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1011, "alarmText" : "PERFORMANCE TEST ALARM 11", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1012, "alarmText" : "PERFORMANCE TEST ALARM 12", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1013, "alarmText" : "PERFORMANCE TEST ALARM 13", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1014, "alarmText" : "PERFORMANCE TEST ALARM 14", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1015, "alarmText" : "PERFORMANCE TEST ALARM 15", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1016, "alarmText" : "PERFORMANCE TEST ALARM 16", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1017, "alarmText" : "PERFORMANCE TEST ALARM 17", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1018, "alarmText" : "PERFORMANCE TEST ALARM 18", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1019, "alarmText" : "PERFORMANCE TEST ALARM 19", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1020, "alarmText" : "PERFORMANCE TEST ALARM 20", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1021, "alarmText" : "PERFORMANCE TEST ALARM 21", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1022, "alarmText" : "PERFORMANCE TEST ALARM 22", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1023, "alarmText" : "PERFORMANCE TEST ALARM 23", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1024, "alarmText" : "PERFORMANCE TEST ALARM 24", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1025, "alarmText" : "PERFORMANCE TEST ALARM 25", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1026, "alarmText" : "PERFORMANCE TEST ALARM 26", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1027, "alarmText" : "PERFORMANCE TEST ALARM 27", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1028, "alarmText" : "PERFORMANCE TEST ALARM 28", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1029, "alarmText" : "PERFORMANCE TEST ALARM 29", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1030, "alarmText" : "PERFORMANCE TEST ALARM 30", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1031, "alarmText" : "PERFORMANCE TEST ALARM 31", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1032, "alarmText" : "PERFORMANCE TEST ALARM 32", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1033, "alarmText" : "PERFORMANCE TEST ALARM 33", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1034, "alarmText" : "PERFORMANCE TEST ALARM 34", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1035, "alarmText" : "PERFORMANCE TEST ALARM 35", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1036, "alarmText" : "PERFORMANCE TEST ALARM 36", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1037, "alarmText" : "PERFORMANCE TEST ALARM 37", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1038, "alarmText" : "PERFORMANCE TEST ALARM 38", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1039, "alarmText" : "PERFORMANCE TEST ALARM 39", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1040, "alarmText" : "PERFORMANCE TEST ALARM 40", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1041, "alarmText" : "PERFORMANCE TEST ALARM 41", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1042, "alarmText" : "PERFORMANCE TEST ALARM 42", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1043, "alarmText" : "PERFORMANCE TEST ALARM 43", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1044, "alarmText" : "PERFORMANCE TEST ALARM 44", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1045, "alarmText" : "PERFORMANCE TEST ALARM 45", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1046, "alarmText" : "PERFORMANCE TEST ALARM 46", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1047, "alarmText" : "PERFORMANCE TEST ALARM 47", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1048, "alarmText" : "PERFORMANCE TEST ALARM 48", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1049, "alarmText" : "PERFORMANCE TEST ALARM 49", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 }, { "alarmId" : 1050, "alarmText" : "PERFORMANCE TEST ALARM 50", "eventType" : "Performance test", - "operationInstructions" : "Not defined" + "operationInstructions" : "Not defined", + "raiseDelay" : 0, + "clearDelay" : 0 } ] } -- 2.16.6