From 6f73fa33d550b6ff5ca2590757e98cec88254812 Mon Sep 17 00:00:00 2001 From: vipin Date: Tue, 6 Oct 2020 06:51:53 +0000 Subject: [PATCH] - FM performance test tool first version for review - Process synchronization issues resoved in alarm manager when handling many simultaneous posts - Time synschronization issues handled in testcases - Moved the performance test resources from cli to pfresources directory Change-Id: I3f7eac3a1494e7a6abbef7232d025757456ffc4d Signed-off-by: vipin --- alarm/alarm.go | 22 +- alarm/types.go | 51 ++++ build/Dockerfile | 6 + build/build_ubuntu.sh | 2 + build/run.sh | 2 + build/run_local.sh | 2 + cli/alarm-cli.go | 320 +++++++++++++++++++++--- go.mod | 2 + manager/cmd/manager.go | 33 ++- manager/cmd/manager_test.go | 4 +- manager/cmd/restapi.go | 1 + manager/cmd/types.go | 6 + perfresources/perf-alarm-definition.json | 304 +++++++++++++++++++++++ perfresources/perf-alarm-object.json | 404 +++++++++++++++++++++++++++++++ 14 files changed, 1102 insertions(+), 57 deletions(-) create mode 100755 perfresources/perf-alarm-definition.json create mode 100755 perfresources/perf-alarm-object.json diff --git a/alarm/alarm.go b/alarm/alarm.go index 5dfe896..3856fce 100755 --- a/alarm/alarm.go +++ b/alarm/alarm.go @@ -56,8 +56,11 @@ func InitAlarm(mo, id string) (*RICAlarm, error) { } if os.Getenv("ALARM_IF_RMR") == "" { - go InitRMR(r) - } + go InitRMR(r, "") + } else { + go InitRMR(r, ALARM_MANAGER_RMR_URL) + } + return r, nil } @@ -177,14 +180,15 @@ func (r *RICAlarm) ReceiveMessage(cb func(AlarmMessage)) error { return errors.New("rmrRcv failed!") } -func InitRMR(r *RICAlarm) error { +func InitRMR(r *RICAlarm, endpoint string) error { // Setup static RT for alarm system - endpoint := ALARM_MANAGER_RMR_URL - if r.moId == "my-pod" { - endpoint = "127.0.0.1:4560" - } else if r.moId == "my-pod-lib" { - endpoint = "127.0.0.1:4588" - } + if endpoint == "" { + if r.moId == "my-pod" { + endpoint = "127.0.0.1:4560" + } else if r.moId == "my-pod-lib" { + endpoint = "127.0.0.1:4588" + } + } alarmRT := fmt.Sprintf("newrt|start\nrte|13111|%s\nnewrt|end\n", endpoint) alarmRTFile := "/tmp/alarm.rt" diff --git a/alarm/types.go b/alarm/types.go index 5a70f18..be96fec 100755 --- a/alarm/types.go +++ b/alarm/types.go @@ -89,6 +89,56 @@ const ( // Temp alarm constants & definitions const ( + PERFORMANCE_TEST_ALARM_1 int = 1001 + PERFORMANCE_TEST_ALARM_2 int = 1002 + PERFORMANCE_TEST_ALARM_3 int = 1003 + PERFORMANCE_TEST_ALARM_4 int = 1004 + PERFORMANCE_TEST_ALARM_5 int = 1005 + PERFORMANCE_TEST_ALARM_6 int = 1006 + PERFORMANCE_TEST_ALARM_7 int = 1007 + PERFORMANCE_TEST_ALARM_8 int = 1008 + PERFORMANCE_TEST_ALARM_9 int = 1009 + PERFORMANCE_TEST_ALARM_10 int = 1010 + PERFORMANCE_TEST_ALARM_11 int = 1011 + PERFORMANCE_TEST_ALARM_12 int = 1012 + PERFORMANCE_TEST_ALARM_13 int = 1013 + PERFORMANCE_TEST_ALARM_14 int = 1014 + PERFORMANCE_TEST_ALARM_15 int = 1015 + PERFORMANCE_TEST_ALARM_16 int = 1016 + PERFORMANCE_TEST_ALARM_17 int = 1017 + PERFORMANCE_TEST_ALARM_18 int = 1018 + PERFORMANCE_TEST_ALARM_19 int = 1019 + PERFORMANCE_TEST_ALARM_20 int = 1020 + PERFORMANCE_TEST_ALARM_21 int = 1021 + PERFORMANCE_TEST_ALARM_22 int = 1022 + PERFORMANCE_TEST_ALARM_23 int = 1023 + PERFORMANCE_TEST_ALARM_24 int = 1024 + PERFORMANCE_TEST_ALARM_25 int = 1025 + PERFORMANCE_TEST_ALARM_26 int = 1026 + PERFORMANCE_TEST_ALARM_27 int = 1027 + PERFORMANCE_TEST_ALARM_28 int = 1028 + PERFORMANCE_TEST_ALARM_29 int = 1029 + PERFORMANCE_TEST_ALARM_30 int = 1030 + PERFORMANCE_TEST_ALARM_31 int = 1031 + PERFORMANCE_TEST_ALARM_32 int = 1032 + PERFORMANCE_TEST_ALARM_33 int = 1033 + PERFORMANCE_TEST_ALARM_34 int = 1034 + PERFORMANCE_TEST_ALARM_35 int = 1035 + PERFORMANCE_TEST_ALARM_36 int = 1036 + PERFORMANCE_TEST_ALARM_37 int = 1037 + PERFORMANCE_TEST_ALARM_38 int = 1038 + PERFORMANCE_TEST_ALARM_39 int = 1039 + PERFORMANCE_TEST_ALARM_40 int = 1040 + PERFORMANCE_TEST_ALARM_41 int = 1041 + PERFORMANCE_TEST_ALARM_42 int = 1042 + PERFORMANCE_TEST_ALARM_43 int = 1043 + PERFORMANCE_TEST_ALARM_44 int = 1044 + PERFORMANCE_TEST_ALARM_45 int = 1045 + PERFORMANCE_TEST_ALARM_46 int = 1046 + PERFORMANCE_TEST_ALARM_47 int = 1047 + PERFORMANCE_TEST_ALARM_48 int = 1048 + PERFORMANCE_TEST_ALARM_49 int = 1049 + PERFORMANCE_TEST_ALARM_50 int = 1050 RIC_RT_DISTRIBUTION_FAILED int = 8004 TCP_CONNECTIVITY_LOST_TO_DBAAS int = 8005 E2_CONNECTIVITY_LOST_TO_GNODEB int = 8006 @@ -105,6 +155,7 @@ type AlarmDefinition struct { } var RICAlarmDefinitions map[int]*AlarmDefinition +var RICPerfAlarmObjects map[int]*Alarm const ( ALARM_MANAGER_HTTP_URL string = "http://service-ricplt-alarmmanager-http.ricplt:8080" diff --git a/build/Dockerfile b/build/Dockerfile index 6572c49..7e9ebc8 100755 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -35,11 +35,17 @@ FROM ubuntu:18.04 COPY --from=ubuntu-alarm-manager /go/src/am/build/run.sh / COPY --from=ubuntu-alarm-manager /go/src/am/manager/alarm-manager / +COPY --from=ubuntu-alarm-manager /go/src/am/cli/alarm-cli / +COPY --from=ubuntu-alarm-manager /go/src/am/perfresources/perf-alarm-object.json / +COPY --from=ubuntu-alarm-manager /go/src/am/perfresources/perf-alarm-definition.json / COPY --from=ubuntu-alarm-manager /go/src/am/config/* / COPY --from=ubuntu-alarm-manager /usr/local/lib /usr/local/lib COPY --from=ubuntu-alarm-manager /go/src/am/definitions/* / RUN ldconfig +ENV DEF_FILE="./alarm-definition.json" +ENV PERF_DEF_FILE="./perf-alarm-definition.json" +ENV PERF_OBJ_FILE="./perf-alarm-object.json" RUN chmod 755 /run.sh CMD /run.sh diff --git a/build/build_ubuntu.sh b/build/build_ubuntu.sh index 3136eca..79e5a8c 100755 --- a/build/build_ubuntu.sh +++ b/build/build_ubuntu.sh @@ -44,6 +44,8 @@ export RMR_SEED_RT=../config/uta_rtg.rt # xApp stuff export DEF_FILE=../definitions/alarm-definition.json +export PERF_DEF_FILE=../perfresources/perf-alarm-definition.json +export PERF_OBJ_FILE=../perfresources/perf-alarm-object.json GO111MODULE=on GO_ENABLED=0 GOOS=linux diff --git a/build/run.sh b/build/run.sh index 1e1ad85..697de26 100755 --- a/build/run.sh +++ b/build/run.sh @@ -25,5 +25,7 @@ export RMR_SEED_RT=./uta_rtg.rt export RMR_SRC_ID="service-ricplt-alarmmanager-rmr.ricplt" export DEF_FILE=./alarm-definition.json +export PERF_DEF_FILE=./perf-alarm-definition.json +export PERF_OBJ_FILE=./perf-alarm-object.json exec ./alarm-manager -f ./config-file.json diff --git a/build/run_local.sh b/build/run_local.sh index 1c82c74..ddd4524 100755 --- a/build/run_local.sh +++ b/build/run_local.sh @@ -25,5 +25,7 @@ export RMR_SEED_RT=$PWD/config/uta_rtg.rt export RMR_SRC_ID="service-ricplt-alarmmanager-rmr.ricplt" export DEF_FILE=$PWD/definitions/alarm-definition.json +export PERF_DEF_FILE=$PWD/perfresources/perf-alarm-definition.json +export PERF_OBJ_FILE=$PWD/perfresources/perf-alarm-object.json $PWD/manager/alarm-manager -f $PWD/config/config-file.json diff --git a/cli/alarm-cli.go b/cli/alarm-cli.go index 0a1312a..afe9da9 100755 --- a/cli/alarm-cli.go +++ b/cli/alarm-cli.go @@ -13,6 +13,7 @@ import ( "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm" "github.com/jedib0t/go-pretty/table" "github.com/thatisuday/commando" + "sync" ) type CliAlarmDefinitions struct { @@ -23,6 +24,24 @@ type AlarmClient struct { alarmer *alarm.RICAlarm } +type RicPerfAlarmObjects struct { + AlarmObjects []*alarm.Alarm `json:"alarmobjects"` +} + +var CLIPerfAlarmObjects map[int]*alarm.Alarm + +var wg sync.WaitGroup + +var CliPerfAlarmDefinitions CliAlarmDefinitions + +const ( + Raise string = "RAISE" + Clear string = "CLEAR" + End string = "END" + PeakTestDuration int = 60 + OneSecondDuration int = 1 +) + func main() { // configure commando @@ -67,7 +86,7 @@ func main() { AddFlag("port", "Alarm manager host address", commando.String, "8080"). AddFlag("if", "http or rmr used as interface", commando.String, "http"). SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) { - postAlarm(flags, readAlarmParams(flags, false), alarm.AlarmActionRaise) + postAlarm(flags, readAlarmParams(flags, false), alarm.AlarmActionRaise, nil) }) // Clear an alarm @@ -82,7 +101,7 @@ func main() { AddFlag("port", "Alarm manager host address", commando.String, "8080"). AddFlag("if", "http or rmr used as interface", commando.String, "http"). SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) { - postAlarm(flags, readAlarmParams(flags, true), alarm.AlarmActionClear) + postAlarm(flags, readAlarmParams(flags, true), alarm.AlarmActionClear, nil) }) // Configure an alarm manager @@ -119,6 +138,20 @@ func main() { SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) { deleteAlarmDefinition(flags) }) + // Conduct performance test for alarm-go + commando. + Register("perf"). + SetShortDescription("Conduct performance test with given parameters"). + AddFlag("prf", "performance profile id", commando.Int, nil). + AddFlag("nal", "number of alarms", commando.Int, nil). + AddFlag("aps", "alarms per sec", commando.Int, nil). + AddFlag("tim", "total time of test", commando.Int, nil). + AddFlag("host", "Alarm manager host address", commando.String, "localhost"). + AddFlag("port", "Alarm manager host address", commando.String, "8080"). + AddFlag("if", "http or rmr used as interface", commando.String, "http"). + SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) { + conductperformancetest(flags) + }) // parse command-line arguments commando.Parse(nil) @@ -147,7 +180,7 @@ func getAlarms(flags map[string]commando.FlagValue, action alarm.AlarmAction) (a targetUrl := fmt.Sprintf("http://%s:%s/ric/v1/alarms/%s", host, port, action) resp, err := http.Get(targetUrl) if err != nil || resp == nil || resp.Body == nil { - fmt.Println("Couldn't fetch active alarm list due to error: %v", err) + fmt.Println("Couldn't fetch active alarm list due to error: ", err) return alarms } @@ -155,7 +188,7 @@ func getAlarms(flags map[string]commando.FlagValue, action alarm.AlarmAction) (a body, err := ioutil.ReadAll(resp.Body) if err != nil { - fmt.Println("ioutil.ReadAll failed: %v", err) + fmt.Println("ioutil.ReadAll failed: ", err) return alarms } @@ -163,49 +196,58 @@ func getAlarms(flags map[string]commando.FlagValue, action alarm.AlarmAction) (a return alarms } -func postAlarm(flags map[string]commando.FlagValue, a alarm.Alarm, action alarm.AlarmAction) { - - // Check the interface to be used for raise or clear the alarm - rmr_or_http, _ := flags["if"].GetString() - if rmr_or_http == "rmr" { - alarmClient := NewAlarmClient("my-pod", "my-app") - if alarmClient == nil { - return - } - - // Wait until RMR is up-and-running - for !alarmClient.alarmer.IsRMRReady() { - time.Sleep(100 * time.Millisecond) - } +func postAlarmWithRmrIf(a alarm.Alarm, action alarm.AlarmAction, alarmClient *AlarmClient) { + if alarmClient == nil { + alarmClient = NewAlarmClient("my-pod", "my-app") + } + if alarmClient == nil { + return + } - if action == alarm.AlarmActionRaise { - alarmClient.alarmer.Raise(a) - } + // Wait until RMR is up-and-running + for !alarmClient.alarmer.IsRMRReady() { + time.Sleep(100 * time.Millisecond) + } - if action == alarm.AlarmActionClear { - alarmClient.alarmer.Clear(a) - } - return + if action == alarm.AlarmActionRaise { + alarmClient.alarmer.Raise(a) } - host, _ := flags["host"].GetString() - port, _ := flags["port"].GetString() - targetUrl := fmt.Sprintf("http://%s:%s/ric/v1/alarms", host, port) + if action == alarm.AlarmActionClear { + alarmClient.alarmer.Clear(a) + } + return +} +func postAlarmWithHttpIf(targetUrl string, a alarm.Alarm, action alarm.AlarmAction) { m := alarm.AlarmMessage{Alarm: a, AlarmAction: action} jsonData, err := json.Marshal(m) if err != nil { - fmt.Println("json.Marshal failed: %v", err) + fmt.Println("json.Marshal failed: ", err) return } resp, err := http.Post(targetUrl, "application/json", bytes.NewBuffer(jsonData)) if err != nil || resp == nil { - fmt.Println("Couldn't fetch active alarm list due to error: %v", err) + fmt.Println("Couldn't fetch active alarm list due to error: ", err) return } } +func postAlarm(flags map[string]commando.FlagValue, a alarm.Alarm, action alarm.AlarmAction, alarmClient *AlarmClient) { + // Check the interface to be used for raise or clear the alarm + rmr_or_http, _ := flags["if"].GetString() + if rmr_or_http == "rmr" { + postAlarmWithRmrIf(a, action, alarmClient) + } else { + + host, _ := flags["host"].GetString() + port, _ := flags["port"].GetString() + targetUrl := fmt.Sprintf("http://%s:%s/ric/v1/alarms", host, port) + postAlarmWithHttpIf(targetUrl, a, action) + } +} + func displayAlarms(alarms []alarm.AlarmMessage, isHistory bool) { t := table.NewWriter() t.SetOutputMirror(os.Stdout) @@ -242,13 +284,13 @@ func postAlarmConfig(flags map[string]commando.FlagValue) { m := alarm.AlarmConfigParams{MaxActiveAlarms: maxactivealarms, MaxAlarmHistory: maxalarmhistory} jsonData, err := json.Marshal(m) if err != nil { - fmt.Println("json.Marshal failed: %v", err) + fmt.Println("json.Marshal failed: ", err) return } resp, err := http.Post(targetUrl, "application/json", bytes.NewBuffer(jsonData)) if err != nil || resp == nil { - fmt.Println("Couldn't fetch post alarm configuration due to error: %v", err) + fmt.Println("Couldn't fetch post alarm configuration due to error: ", err) return } } @@ -271,13 +313,13 @@ func postAlarmDefinition(flags map[string]commando.FlagValue) { m := CliAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarmdefinition}} jsonData, err := json.Marshal(m) if err != nil { - fmt.Println("json.Marshal failed: %v", err) + fmt.Println("json.Marshal failed: ", err) return } resp, err := http.Post(targetUrl, "application/json", bytes.NewBuffer(jsonData)) if err != nil || resp == nil { - fmt.Println("Couldn't post alarm definition due to error: %v", err) + fmt.Println("Couldn't post alarm definition due to error: ", err) return } } @@ -292,12 +334,12 @@ func deleteAlarmDefinition(flags map[string]commando.FlagValue) { client := &http.Client{} req, err := http.NewRequest("DELETE", targetUrl, nil) if err != nil || req == nil { - fmt.Println("Couldn't make delete request due to error: %v", err) + fmt.Println("Couldn't make delete request due to error: ", err) return } resp, errr := client.Do(req) if errr != nil || resp == nil { - fmt.Println("Couldn't send delete request due to error: %v", err) + fmt.Println("Couldn't send delete request due to error: ", err) return } } @@ -313,3 +355,209 @@ func NewAlarmClient(moId, appId string) *AlarmClient { fmt.Println("Failed to create alarmInstance", err) return nil } + +// Conduct performance testing +func conductperformancetest(flags map[string]commando.FlagValue) { + var readerror error + var senderror error + var readobjerror error + host, _ := flags["host"].GetString() + port, _ := flags["port"].GetString() + targetUrl := fmt.Sprintf("http://%s:%s/ric/v1/alarms/define", host, port) + readerror = readPerfAlarmDefinitionFromJson() + if readerror == nil { + senderror = sendPerfAlarmDefinitionToAlarmManager(targetUrl) + if senderror == nil { + fmt.Println("sent performance alarm definitions to alarm manager") + CLIPerfAlarmObjects = make(map[int]*alarm.Alarm) + readobjerror = readPerfAlarmObjectFromJson() + if readobjerror == nil { + profile, _ := flags["prf"].GetInt() + if profile == 1 { + fmt.Println("starting peak performance test") + peakPerformanceTest(flags) + } else if profile == 2 { + fmt.Println("starting endurance test") + enduranceTest(flags) + } else { + fmt.Println("Unknown profile, received profile = ", profile) + } + } else { + fmt.Println("reading performance alarm objects from json file failed ") + } + } else { + fmt.Println("sending performance alarm definitions to alarm manager failed ") + } + + } else { + fmt.Println("reading performance alarm definitions from json file failed ") + } + +} + +func peakPerformanceTest(flags map[string]commando.FlagValue) { + nalarms, _ := flags["nal"].GetInt() + var count int = 0 + for aid, obj := range CLIPerfAlarmObjects { + count = count + 1 + if count <= nalarms { + fmt.Println("peakPerformanceTest: invoking worker routine ", count, aid, *obj) + wg.Add(1) + go raiseClearAlarmOnce(obj, flags) + } else { + break + } + } + fmt.Println("peakPerformanceTest: Waiting for workers to finish") + wg.Wait() + fmt.Println("peakPerformanceTest: Wait completed") +} + +func enduranceTest(flags map[string]commando.FlagValue) { + alarmspersec, _ := flags["aps"].GetInt() + var count int = 0 + for aid, obj := range CLIPerfAlarmObjects { + count = count + 1 + if count <= alarmspersec { + fmt.Println("enduranceTest: invoking worker routine ", count, aid, *obj) + wg.Add(1) + go raiseClearAlarmOverPeriod(obj, flags) + } else { + break + } + } + fmt.Println("enduranceTest: Waiting for workers to finish") + wg.Wait() + fmt.Println("enduranceTest: Wait completed") +} + +func readPerfAlarmObjectFromJson() error { + filename := os.Getenv("PERF_OBJ_FILE") + file, err := ioutil.ReadFile(filename) + if err == nil { + data := RicPerfAlarmObjects{} + err = json.Unmarshal([]byte(file), &data) + if err == nil { + for _, alarmObject := range data.AlarmObjects { + ricAlarmObject := new(alarm.Alarm) + ricAlarmObject.ManagedObjectId = alarmObject.ManagedObjectId + ricAlarmObject.ApplicationId = alarmObject.ApplicationId + ricAlarmObject.SpecificProblem = alarmObject.SpecificProblem + ricAlarmObject.PerceivedSeverity = alarmObject.PerceivedSeverity + ricAlarmObject.AdditionalInfo = alarmObject.AdditionalInfo + ricAlarmObject.IdentifyingInfo = alarmObject.IdentifyingInfo + CLIPerfAlarmObjects[alarmObject.SpecificProblem] = ricAlarmObject + } + } else { + fmt.Println("readPerfAlarmObjectFromJson: json.Unmarshal failed with error ", err) + return err + } + } else { + fmt.Println("readPerfAlarmObjectFromJson: ioutil.ReadFile failed with error ", err) + return err + } + return nil +} + +func readPerfAlarmDefinitionFromJson() error { + filename := os.Getenv("PERF_DEF_FILE") + file, err := ioutil.ReadFile(filename) + if err == nil { + data := CliAlarmDefinitions{} + err = json.Unmarshal([]byte(file), &data) + if err == nil { + for _, alarmDefinition := range data.AlarmDefinitions { + _, exists := alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] + if exists { + fmt.Println("ReadPerfAlarmDefinitionFromJson: alarm definition already exists for ", alarmDefinition.AlarmId) + } else { + fmt.Println("ReadPerfAlarmDefinitionFromJson: alarm ", alarmDefinition.AlarmId) + ricAlarmDefintion := new(alarm.AlarmDefinition) + ricAlarmDefintion.AlarmId = alarmDefinition.AlarmId + ricAlarmDefintion.AlarmText = alarmDefinition.AlarmText + ricAlarmDefintion.EventType = alarmDefinition.EventType + ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions + CliPerfAlarmDefinitions.AlarmDefinitions = append(CliPerfAlarmDefinitions.AlarmDefinitions, ricAlarmDefintion) + } + } + } else { + fmt.Println("ReadPerfAlarmDefinitionFromJson: json.Unmarshal failed with error: ", err) + return err + } + } else { + fmt.Println("ReadPerfAlarmDefinitionFromJson: ioutil.ReadFile failed with error: ", err) + return err + } + return nil +} + +func sendPerfAlarmDefinitionToAlarmManager(targetUrl string) error { + + jsonData, err := json.Marshal(CliPerfAlarmDefinitions) + if err != nil { + fmt.Println("sendPerfAlarmDefinitionToAlarmManager: json.Marshal failed: ", err) + return err + } + + resp, err := http.Post(targetUrl, "application/json", bytes.NewBuffer(jsonData)) + if err != nil || resp == nil { + fmt.Println("sendPerfAlarmDefinitionToAlarmManager: Couldn't post alarm definition to targeturl due to error: ", targetUrl, err) + return err + } + return nil +} + +func wakeUpAfterTime(timeinseconds int, chn chan string, action string) { + time.Sleep(time.Second * time.Duration(timeinseconds)) + chn <- action +} + +func raiseClearAlarmOnce(alarmobject *alarm.Alarm, flags map[string]commando.FlagValue) { + var alarmClient *AlarmClient = nil + defer wg.Done() + chn := make(chan string, 1) + rmr_or_http, _ := flags["if"].GetString() + if rmr_or_http == "rmr" { + alarmClient = NewAlarmClient("my-pod", "my-app") + } + postAlarm(flags, *alarmobject, alarm.AlarmActionRaise, alarmClient) + go wakeUpAfterTime(PeakTestDuration, chn, Clear) + select { + case res := <-chn: + if res == Clear { + postAlarm(flags, *alarmobject, alarm.AlarmActionClear, alarmClient) + go wakeUpAfterTime(PeakTestDuration, chn, End) + } else if res == End { + return + } + } +} + +func raiseClearAlarmOverPeriod(alarmobject *alarm.Alarm, flags map[string]commando.FlagValue) { + var alarmClient *AlarmClient = nil + defer wg.Done() + timeinminutes, _ := flags["tim"].GetInt() + timeinseconds := timeinminutes * 60 + chn := make(chan string, 1) + rmr_or_http, _ := flags["if"].GetString() + if rmr_or_http == "rmr" { + alarmClient = NewAlarmClient("my-pod", "my-app") + } + postAlarm(flags, *alarmobject, alarm.AlarmActionRaise, alarmClient) + go wakeUpAfterTime(OneSecondDuration, chn, Clear) + go wakeUpAfterTime(timeinseconds, chn, End) + for { + select { + case res := <-chn: + if res == Raise { + postAlarm(flags, *alarmobject, alarm.AlarmActionRaise, alarmClient) + go wakeUpAfterTime(OneSecondDuration, chn, Clear) + } else if res == Clear { + postAlarm(flags, *alarmobject, alarm.AlarmActionClear, alarmClient) + go wakeUpAfterTime(OneSecondDuration, chn, Raise) + } else if res == End { + return + } + } + } +} diff --git a/go.mod b/go.mod index b6ae937..0fb43fa 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,8 @@ require ( github.com/gorilla/mux v1.7.1 github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/prometheus/alertmanager v0.20.0 github.com/spf13/viper v1.6.2 github.com/stretchr/testify v1.5.1 diff --git a/manager/cmd/manager.go b/manager/cmd/manager.go index 0cb528f..f00a812 100755 --- a/manager/cmd/manager.go +++ b/manager/cmd/manager.go @@ -75,8 +75,10 @@ func (a *AlarmManager) HandleAlarms(rp *app.RMRParams) (*alert.PostAlertsOK, err } func (a *AlarmManager) ProcessAlarm(m *alarm.AlarmMessage) (*alert.PostAlertsOK, error) { + a.mutex.Lock() if _, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok { app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem) + a.mutex.Unlock() return nil, nil } @@ -86,6 +88,7 @@ func (a *AlarmManager) ProcessAlarm(m *alarm.AlarmMessage) (*alert.PostAlertsOK, app.Logger.Info("Duplicate alarm found, suppressing ...") if m.PerceivedSeverity == a.activeAlarms[idx].PerceivedSeverity { // Duplicate with same severity found + a.mutex.Unlock() return nil, nil } else { // Remove duplicate with different severity @@ -98,27 +101,37 @@ func (a *AlarmManager) ProcessAlarm(m *alarm.AlarmMessage) (*alert.PostAlertsOK, if found { a.alarmHistory = append(a.alarmHistory, *m) a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active") - if len(a.alarmHistory) >= a.maxAlarmHistory { + if ((len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false)){ app.Logger.Error("alarm history count exceeded maxAlarmHistory threshold") histAlarm := a.alarmClient.NewAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, alarm.SeverityWarning, "threshold", "history") histAlarmMessage := alarm.AlarmMessage{Alarm: histAlarm, AlarmAction: alarm.AlarmActionRaise, AlarmTime: (time.Now().UnixNano())} a.activeAlarms = append(a.activeAlarms, histAlarmMessage) a.alarmHistory = append(a.alarmHistory, histAlarmMessage) } + if ((a.exceededActiveAlarmOn == true) && (m.Alarm.SpecificProblem == alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD)) { + a.exceededActiveAlarmOn = false + } + if ((a.exceededAlarmHistoryOn == true) && (m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD)) { + a.exceededAlarmHistoryOn = false + } if a.postClear { + a.mutex.Unlock() return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusResolved, m.AlarmTime)) } } app.Logger.Info("No matching active alarm found, suppressing ...") + a.mutex.Unlock() return nil, nil } // New alarm -> update active alarms and post to Alert Manager if m.AlarmAction == alarm.AlarmActionRaise { a.UpdateAlarmLists(m) + a.mutex.Unlock() return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime)) } + a.mutex.Unlock() return nil, nil } @@ -133,34 +146,30 @@ func (a *AlarmManager) IsMatchFound(newAlarm alarm.Alarm) (int, bool) { } func (a *AlarmManager) RemoveAlarm(alarms []alarm.AlarmMessage, i int, listName string) []alarm.AlarmMessage { - a.mutex.Lock() - defer a.mutex.Unlock() - app.Logger.Info("Alarm '%+v' deleted from the '%s' list", alarms[i], listName) copy(alarms[i:], alarms[i+1:]) return alarms[:len(alarms)-1] } func (a *AlarmManager) UpdateAlarmLists(newAlarm *alarm.AlarmMessage) { - a.mutex.Lock() - defer a.mutex.Unlock() - /* If maximum number of active alarms is reached, an error log writing is made, and new alarm indicating the problem is raised. The attempt to raise the alarm next time will be supressed when found as duplicate. */ - if len(a.activeAlarms) >= a.maxActiveAlarms { + if ((len(a.activeAlarms) >= a.maxActiveAlarms) && (a.exceededActiveAlarmOn == false)) { app.Logger.Error("active alarm count exceeded maxActiveAlarms threshold") actAlarm := a.alarmClient.NewAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, alarm.SeverityWarning, "threshold", "active") actAlarmMessage := alarm.AlarmMessage{Alarm: actAlarm, AlarmAction: alarm.AlarmActionRaise, AlarmTime: (time.Now().UnixNano())} a.activeAlarms = append(a.activeAlarms, actAlarmMessage) a.alarmHistory = append(a.alarmHistory, actAlarmMessage) + a.exceededActiveAlarmOn = true } - if len(a.alarmHistory) >= a.maxAlarmHistory { + if ((len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false)) { app.Logger.Error("alarm history count exceeded maxAlarmHistory threshold") histAlarm := a.alarmClient.NewAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, alarm.SeverityWarning, "threshold", "history") histAlarmMessage := alarm.AlarmMessage{Alarm: histAlarm, AlarmAction: alarm.AlarmActionRaise, AlarmTime: (time.Now().UnixNano())} a.activeAlarms = append(a.activeAlarms, histAlarmMessage) a.alarmHistory = append(a.alarmHistory, histAlarmMessage) + a.exceededAlarmHistoryOn = true } // @todo: For now just keep the alarms (both active and history) in-memory. Use SDL later for persistence @@ -258,10 +267,10 @@ func (a *AlarmManager) ReadAlarmDefinitionFromJson() { } } } else { - app.Logger.Error("json.Unmarshal failed with error %v", err) + app.Logger.Error("ReadAlarmDefinitionFromJson: json.Unmarshal failed with error %v", err) } } else { - app.Logger.Error("ioutil.ReadFile failed with error %v", err) + app.Logger.Error("ReadAlarmDefinitionFromJson: ioutil.ReadFile failed with error %v", err) } } @@ -312,6 +321,8 @@ func NewAlarmManager(amHost string, alertInterval int) *AlarmManager { alarmHistory: make([]alarm.AlarmMessage, 0), maxActiveAlarms: app.Config.GetInt("controls.maxActiveAlarms"), maxAlarmHistory: app.Config.GetInt("controls.maxAlarmHistory"), + exceededActiveAlarmOn: false, + exceededAlarmHistoryOn: false, } } diff --git a/manager/cmd/manager_test.go b/manager/cmd/manager_test.go index f4a5aee..34b3902 100755 --- a/manager/cmd/manager_test.go +++ b/manager/cmd/manager_test.go @@ -48,6 +48,7 @@ var eventChan chan string // Test cases func TestMain(M *testing.M) { alarmManager = NewAlarmManager("localhost:9093", 500) + alarmManager.alertInterval = 20000 go alarmManager.Run(false) time.Sleep(time.Duration(10) * time.Second) @@ -238,6 +239,7 @@ func TestMultipleAlarmsRaisedSucess(t *testing.T) { b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11") assert.Nil(t, alarmer.Raise(b), "raise failed") + time.Sleep(time.Duration(2) * time.Second) VerifyAlarm(t, a, 2) VerifyAlarm(t, b, 2) } @@ -332,7 +334,7 @@ func TestActiveAlarmMaxThresholds(t *testing.T) { func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string { receivedAlert := waitForEvent() - assert.Equal(t, len(alarmManager.activeAlarms), expectedCount) + assert.Equal(t, expectedCount, len(alarmManager.activeAlarms)) _, ok := alarmManager.IsMatchFound(a) assert.True(t, ok) diff --git a/manager/cmd/restapi.go b/manager/cmd/restapi.go index 7a09083..80f4997 100755 --- a/manager/cmd/restapi.go +++ b/manager/cmd/restapi.go @@ -96,6 +96,7 @@ func (a *AlarmManager) SetAlarmDefinition(w http.ResponseWriter, r *http.Request ricAlarmDefintion.EventType = alarmDefinition.EventType ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion + app.Logger.Debug("POST - alarm definition added for alarm id %v", alarmDefinition.AlarmId) } } diff --git a/manager/cmd/types.go b/manager/cmd/types.go index 73daf48..c415fda 100755 --- a/manager/cmd/types.go +++ b/manager/cmd/types.go @@ -39,6 +39,8 @@ type AlarmManager struct { maxActiveAlarms int maxAlarmHistory int alarmClient *alarm.RICAlarm + exceededActiveAlarmOn bool + exceededAlarmHistoryOn bool } type AlertStatus string @@ -54,3 +56,7 @@ var Hash string type RicAlarmDefinitions struct { AlarmDefinitions []*alarm.AlarmDefinition `json:"alarmdefinitions"` } + +type RicPerfAlarmObjects struct { + AlarmObjects []*alarm.Alarm `json:"alarmobjects"` +} diff --git a/perfresources/perf-alarm-definition.json b/perfresources/perf-alarm-definition.json new file mode 100755 index 0000000..e12fb0f --- /dev/null +++ b/perfresources/perf-alarm-definition.json @@ -0,0 +1,304 @@ +{ + "alarmdefinitions" : [ + { + "alarmId" : 1001, + "alarmText" : "PERFORMANCE TEST ALARM 1", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1002, + "alarmText" : "PERFORMANCE TEST ALARM 2", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1003, + "alarmText" : "PERFORMANCE TEST ALARM 3", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1004, + "alarmText" : "PERFORMANCE TEST ALARM 4", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1005, + "alarmText" : "PERFORMANCE TEST ALARM 5", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1006, + "alarmText" : "PERFORMANCE TEST ALARM 6", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1007, + "alarmText" : "PERFORMANCE TEST ALARM 7", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1008, + "alarmText" : "PERFORMANCE TEST ALARM 8", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1009, + "alarmText" : "PERFORMANCE TEST ALARM 9", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1010, + "alarmText" : "PERFORMANCE TEST ALARM 10", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1011, + "alarmText" : "PERFORMANCE TEST ALARM 11", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1012, + "alarmText" : "PERFORMANCE TEST ALARM 12", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1013, + "alarmText" : "PERFORMANCE TEST ALARM 13", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1014, + "alarmText" : "PERFORMANCE TEST ALARM 14", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1015, + "alarmText" : "PERFORMANCE TEST ALARM 15", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1016, + "alarmText" : "PERFORMANCE TEST ALARM 16", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1017, + "alarmText" : "PERFORMANCE TEST ALARM 17", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1018, + "alarmText" : "PERFORMANCE TEST ALARM 18", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1019, + "alarmText" : "PERFORMANCE TEST ALARM 19", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1020, + "alarmText" : "PERFORMANCE TEST ALARM 20", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1021, + "alarmText" : "PERFORMANCE TEST ALARM 21", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1022, + "alarmText" : "PERFORMANCE TEST ALARM 22", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1023, + "alarmText" : "PERFORMANCE TEST ALARM 23", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1024, + "alarmText" : "PERFORMANCE TEST ALARM 24", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1025, + "alarmText" : "PERFORMANCE TEST ALARM 25", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1026, + "alarmText" : "PERFORMANCE TEST ALARM 26", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1027, + "alarmText" : "PERFORMANCE TEST ALARM 27", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1028, + "alarmText" : "PERFORMANCE TEST ALARM 28", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1029, + "alarmText" : "PERFORMANCE TEST ALARM 29", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1030, + "alarmText" : "PERFORMANCE TEST ALARM 30", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1031, + "alarmText" : "PERFORMANCE TEST ALARM 31", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1032, + "alarmText" : "PERFORMANCE TEST ALARM 32", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1033, + "alarmText" : "PERFORMANCE TEST ALARM 33", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1034, + "alarmText" : "PERFORMANCE TEST ALARM 34", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1035, + "alarmText" : "PERFORMANCE TEST ALARM 35", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1036, + "alarmText" : "PERFORMANCE TEST ALARM 36", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1037, + "alarmText" : "PERFORMANCE TEST ALARM 37", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1038, + "alarmText" : "PERFORMANCE TEST ALARM 38", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1039, + "alarmText" : "PERFORMANCE TEST ALARM 39", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1040, + "alarmText" : "PERFORMANCE TEST ALARM 40", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1041, + "alarmText" : "PERFORMANCE TEST ALARM 41", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1042, + "alarmText" : "PERFORMANCE TEST ALARM 42", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1043, + "alarmText" : "PERFORMANCE TEST ALARM 43", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1044, + "alarmText" : "PERFORMANCE TEST ALARM 44", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1045, + "alarmText" : "PERFORMANCE TEST ALARM 45", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1046, + "alarmText" : "PERFORMANCE TEST ALARM 46", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1047, + "alarmText" : "PERFORMANCE TEST ALARM 47", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1048, + "alarmText" : "PERFORMANCE TEST ALARM 48", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1049, + "alarmText" : "PERFORMANCE TEST ALARM 49", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + }, + { + "alarmId" : 1050, + "alarmText" : "PERFORMANCE TEST ALARM 50", + "eventType" : "Performance test", + "operationInstructions" : "Not defined" + } + ] +} diff --git a/perfresources/perf-alarm-object.json b/perfresources/perf-alarm-object.json new file mode 100755 index 0000000..0c6734a --- /dev/null +++ b/perfresources/perf-alarm-object.json @@ -0,0 +1,404 @@ +{ + "alarmobjects" : [ + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP1", + "specificProblem" : 1001, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP2", + "specificProblem" : 1002, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP3", + "specificProblem" : 1003, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP4", + "specificProblem" : 1004, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP5", + "specificProblem" : 1005, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP6", + "specificProblem" : 1006, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP7", + "specificProblem" : 1007, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP8", + "specificProblem" : 1008, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP9", + "specificProblem" : 1009, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP10", + "specificProblem" : 1010, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP11", + "specificProblem" : 1011, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP12", + "specificProblem" : 1012, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP13", + "specificProblem" : 1013, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP14", + "specificProblem" : 1014, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP15", + "specificProblem" : 1015, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP16", + "specificProblem" : 1016, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP17", + "specificProblem" : 1017, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP18", + "specificProblem" : 1018, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP19", + "specificProblem" : 1019, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP20", + "specificProblem" : 1020, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP21", + "specificProblem" : 1021, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP22", + "specificProblem" : 1022, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP23", + "specificProblem" : 1023, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP24", + "specificProblem" : 1024, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP25", + "specificProblem" : 1025, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP26", + "specificProblem" : 1026, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP27", + "specificProblem" : 1027, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP28", + "specificProblem" : 1028, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP29", + "specificProblem" : 1029, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP30", + "specificProblem" : 1030, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP31", + "specificProblem" : 1031, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP32", + "specificProblem" : 1032, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP33", + "specificProblem" : 1033, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP34", + "specificProblem" : 1034, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP35", + "specificProblem" : 1035, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP36", + "specificProblem" : 1036, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP37", + "specificProblem" : 1037, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP38", + "specificProblem" : 1038, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP39", + "specificProblem" : 1039, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP40", + "specificProblem" : 1040, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP41", + "specificProblem" : 1041, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP42", + "specificProblem" : 1042, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP43", + "specificProblem" : 1043, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP44", + "specificProblem" : 1044, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP45", + "specificProblem" : 1045, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP46", + "specificProblem" : 1046, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP47", + "specificProblem" : 1047, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP48", + "specificProblem" : 1048, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP49", + "specificProblem" : 1049, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + }, + { + "managedObjectId" : "SEP", + "applicationId" : "PERFAPP50", + "specificProblem" : 1050, + "perceivedSeverity" : "WARNING", + "additionalInfo" : "capacity", + "identifyingInfo" : "performance" + } + ] +} -- 2.16.6