- FM performance test tool first version for review 11/4811/4
authorvipin <vipin.mavila@nokia.com>
Tue, 6 Oct 2020 06:51:53 +0000 (06:51 +0000)
committervipin <vipin.mavila@nokia.com>
Fri, 9 Oct 2020 09:07:10 +0000 (09:07 +0000)
- Process synchronization issues resoved in alarm
  manager when handling many simultaneous posts
- Time synschronization issues handled in testcases
- Moved the performance test resources from cli to pfresources directory

Change-Id: I3f7eac3a1494e7a6abbef7232d025757456ffc4d
Signed-off-by: vipin <vipin.mavila@nokia.com>
14 files changed:
alarm/alarm.go
alarm/types.go
build/Dockerfile
build/build_ubuntu.sh
build/run.sh
build/run_local.sh
cli/alarm-cli.go
go.mod
manager/cmd/manager.go
manager/cmd/manager_test.go
manager/cmd/restapi.go
manager/cmd/types.go
perfresources/perf-alarm-definition.json [new file with mode: 0755]
perfresources/perf-alarm-object.json [new file with mode: 0755]

index 5dfe896..3856fce 100755 (executable)
@@ -56,8 +56,11 @@ func InitAlarm(mo, id string) (*RICAlarm, error) {
        }
 
        if os.Getenv("ALARM_IF_RMR") == "" {
-               go InitRMR(r)
-       }
+                go InitRMR(r, "")
+        } else {
+                go InitRMR(r, ALARM_MANAGER_RMR_URL)
+        }
+
        return r, nil
 }
 
@@ -177,14 +180,15 @@ func (r *RICAlarm) ReceiveMessage(cb func(AlarmMessage)) error {
        return errors.New("rmrRcv failed!")
 }
 
-func InitRMR(r *RICAlarm) error {
+func InitRMR(r *RICAlarm,  endpoint string) error {
        // Setup static RT for alarm system
-       endpoint := ALARM_MANAGER_RMR_URL
-       if r.moId == "my-pod" {
-               endpoint = "127.0.0.1:4560"
-       } else if r.moId == "my-pod-lib" {
-               endpoint = "127.0.0.1:4588"
-       }
+       if endpoint == "" {
+               if r.moId == "my-pod" {
+                        endpoint = "127.0.0.1:4560"
+                } else if r.moId == "my-pod-lib" {
+                        endpoint = "127.0.0.1:4588"
+                }
+        }
 
        alarmRT := fmt.Sprintf("newrt|start\nrte|13111|%s\nnewrt|end\n", endpoint)
        alarmRTFile := "/tmp/alarm.rt"
index 5a70f18..be96fec 100755 (executable)
@@ -89,6 +89,56 @@ const (
 
 // Temp alarm constants & definitions
 const (
+       PERFORMANCE_TEST_ALARM_1           int = 1001
+       PERFORMANCE_TEST_ALARM_2           int = 1002
+       PERFORMANCE_TEST_ALARM_3           int = 1003
+       PERFORMANCE_TEST_ALARM_4           int = 1004
+       PERFORMANCE_TEST_ALARM_5           int = 1005
+       PERFORMANCE_TEST_ALARM_6           int = 1006
+       PERFORMANCE_TEST_ALARM_7           int = 1007
+       PERFORMANCE_TEST_ALARM_8           int = 1008
+       PERFORMANCE_TEST_ALARM_9           int = 1009
+       PERFORMANCE_TEST_ALARM_10          int = 1010
+       PERFORMANCE_TEST_ALARM_11          int = 1011
+       PERFORMANCE_TEST_ALARM_12          int = 1012
+       PERFORMANCE_TEST_ALARM_13          int = 1013
+       PERFORMANCE_TEST_ALARM_14          int = 1014
+       PERFORMANCE_TEST_ALARM_15          int = 1015
+       PERFORMANCE_TEST_ALARM_16          int = 1016
+       PERFORMANCE_TEST_ALARM_17          int = 1017
+       PERFORMANCE_TEST_ALARM_18          int = 1018
+       PERFORMANCE_TEST_ALARM_19          int = 1019
+       PERFORMANCE_TEST_ALARM_20          int = 1020
+       PERFORMANCE_TEST_ALARM_21          int = 1021
+       PERFORMANCE_TEST_ALARM_22          int = 1022
+       PERFORMANCE_TEST_ALARM_23          int = 1023
+       PERFORMANCE_TEST_ALARM_24          int = 1024
+       PERFORMANCE_TEST_ALARM_25          int = 1025
+       PERFORMANCE_TEST_ALARM_26          int = 1026
+       PERFORMANCE_TEST_ALARM_27          int = 1027
+       PERFORMANCE_TEST_ALARM_28          int = 1028
+       PERFORMANCE_TEST_ALARM_29          int = 1029
+       PERFORMANCE_TEST_ALARM_30          int = 1030
+       PERFORMANCE_TEST_ALARM_31          int = 1031
+       PERFORMANCE_TEST_ALARM_32          int = 1032
+       PERFORMANCE_TEST_ALARM_33          int = 1033
+       PERFORMANCE_TEST_ALARM_34          int = 1034
+       PERFORMANCE_TEST_ALARM_35          int = 1035
+       PERFORMANCE_TEST_ALARM_36          int = 1036
+       PERFORMANCE_TEST_ALARM_37          int = 1037
+       PERFORMANCE_TEST_ALARM_38          int = 1038
+       PERFORMANCE_TEST_ALARM_39          int = 1039
+       PERFORMANCE_TEST_ALARM_40          int = 1040
+       PERFORMANCE_TEST_ALARM_41          int = 1041
+       PERFORMANCE_TEST_ALARM_42          int = 1042
+       PERFORMANCE_TEST_ALARM_43          int = 1043
+       PERFORMANCE_TEST_ALARM_44          int = 1044
+       PERFORMANCE_TEST_ALARM_45          int = 1045
+       PERFORMANCE_TEST_ALARM_46          int = 1046
+       PERFORMANCE_TEST_ALARM_47          int = 1047
+       PERFORMANCE_TEST_ALARM_48          int = 1048
+       PERFORMANCE_TEST_ALARM_49          int = 1049
+       PERFORMANCE_TEST_ALARM_50          int = 1050
        RIC_RT_DISTRIBUTION_FAILED         int = 8004
        TCP_CONNECTIVITY_LOST_TO_DBAAS     int = 8005
        E2_CONNECTIVITY_LOST_TO_GNODEB     int = 8006
@@ -105,6 +155,7 @@ type AlarmDefinition struct {
 }
 
 var RICAlarmDefinitions map[int]*AlarmDefinition
+var RICPerfAlarmObjects map[int]*Alarm
 
 const (
        ALARM_MANAGER_HTTP_URL string = "http://service-ricplt-alarmmanager-http.ricplt:8080"
index 6572c49..7e9ebc8 100755 (executable)
@@ -35,11 +35,17 @@ FROM ubuntu:18.04
 
 COPY --from=ubuntu-alarm-manager /go/src/am/build/run.sh /
 COPY --from=ubuntu-alarm-manager /go/src/am/manager/alarm-manager /
+COPY --from=ubuntu-alarm-manager /go/src/am/cli/alarm-cli /
+COPY --from=ubuntu-alarm-manager /go/src/am/perfresources/perf-alarm-object.json /
+COPY --from=ubuntu-alarm-manager /go/src/am/perfresources/perf-alarm-definition.json /
 COPY --from=ubuntu-alarm-manager /go/src/am/config/* /
 COPY --from=ubuntu-alarm-manager /usr/local/lib /usr/local/lib
 COPY --from=ubuntu-alarm-manager /go/src/am/definitions/* / 
 
 RUN ldconfig
+ENV DEF_FILE="./alarm-definition.json"
+ENV PERF_DEF_FILE="./perf-alarm-definition.json"
+ENV PERF_OBJ_FILE="./perf-alarm-object.json"
 
 RUN chmod 755 /run.sh
 CMD /run.sh
index 3136eca..79e5a8c 100755 (executable)
@@ -44,6 +44,8 @@ export RMR_SEED_RT=../config/uta_rtg.rt
 
 # xApp stuff
 export DEF_FILE=../definitions/alarm-definition.json
+export PERF_DEF_FILE=../perfresources/perf-alarm-definition.json
+export PERF_OBJ_FILE=../perfresources/perf-alarm-object.json
 
 GO111MODULE=on GO_ENABLED=0 GOOS=linux
 
index 1e1ad85..697de26 100755 (executable)
@@ -25,5 +25,7 @@
 export RMR_SEED_RT=./uta_rtg.rt
 export RMR_SRC_ID="service-ricplt-alarmmanager-rmr.ricplt"
 export DEF_FILE=./alarm-definition.json
+export PERF_DEF_FILE=./perf-alarm-definition.json
+export PERF_OBJ_FILE=./perf-alarm-object.json
 
 exec ./alarm-manager -f ./config-file.json
index 1c82c74..ddd4524 100755 (executable)
@@ -25,5 +25,7 @@
 export RMR_SEED_RT=$PWD/config/uta_rtg.rt
 export RMR_SRC_ID="service-ricplt-alarmmanager-rmr.ricplt"
 export DEF_FILE=$PWD/definitions/alarm-definition.json
+export PERF_DEF_FILE=$PWD/perfresources/perf-alarm-definition.json
+export PERF_OBJ_FILE=$PWD/perfresources/perf-alarm-object.json
 
 $PWD/manager/alarm-manager -f $PWD/config/config-file.json
index 0a1312a..afe9da9 100755 (executable)
@@ -13,6 +13,7 @@ import (
        "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
        "github.com/jedib0t/go-pretty/table"
        "github.com/thatisuday/commando"
+       "sync"
 )
 
 type CliAlarmDefinitions struct {
@@ -23,6 +24,24 @@ type AlarmClient struct {
        alarmer *alarm.RICAlarm
 }
 
+type RicPerfAlarmObjects struct {
+       AlarmObjects []*alarm.Alarm `json:"alarmobjects"`
+}
+
+var CLIPerfAlarmObjects map[int]*alarm.Alarm
+
+var wg sync.WaitGroup
+
+var CliPerfAlarmDefinitions CliAlarmDefinitions
+
+const (
+       Raise             string = "RAISE"
+       Clear             string = "CLEAR"
+       End               string = "END"
+       PeakTestDuration  int    = 60
+       OneSecondDuration int    = 1
+)
+
 func main() {
 
        // configure commando
@@ -67,7 +86,7 @@ func main() {
                AddFlag("port", "Alarm manager host address", commando.String, "8080").
                AddFlag("if", "http or rmr used as interface", commando.String, "http").
                SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) {
-                       postAlarm(flags, readAlarmParams(flags, false), alarm.AlarmActionRaise)
+                       postAlarm(flags, readAlarmParams(flags, false), alarm.AlarmActionRaise, nil)
                })
 
        // Clear an alarm
@@ -82,7 +101,7 @@ func main() {
                AddFlag("port", "Alarm manager host address", commando.String, "8080").
                AddFlag("if", "http or rmr used as interface", commando.String, "http").
                SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) {
-                       postAlarm(flags, readAlarmParams(flags, true), alarm.AlarmActionClear)
+                       postAlarm(flags, readAlarmParams(flags, true), alarm.AlarmActionClear, nil)
                })
 
        // Configure an alarm manager
@@ -119,6 +138,20 @@ func main() {
                SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) {
                        deleteAlarmDefinition(flags)
                })
+               // Conduct performance test for alarm-go
+       commando.
+               Register("perf").
+               SetShortDescription("Conduct performance test with given parameters").
+               AddFlag("prf", "performance profile id", commando.Int, nil).
+               AddFlag("nal", "number of alarms", commando.Int, nil).
+               AddFlag("aps", "alarms per sec", commando.Int, nil).
+               AddFlag("tim", "total time of test", commando.Int, nil).
+               AddFlag("host", "Alarm manager host address", commando.String, "localhost").
+               AddFlag("port", "Alarm manager host address", commando.String, "8080").
+               AddFlag("if", "http or rmr used as interface", commando.String, "http").
+               SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) {
+                       conductperformancetest(flags)
+               })
 
        // parse command-line arguments
        commando.Parse(nil)
@@ -147,7 +180,7 @@ func getAlarms(flags map[string]commando.FlagValue, action alarm.AlarmAction) (a
        targetUrl := fmt.Sprintf("http://%s:%s/ric/v1/alarms/%s", host, port, action)
        resp, err := http.Get(targetUrl)
        if err != nil || resp == nil || resp.Body == nil {
-               fmt.Println("Couldn't fetch active alarm list due to error: %v", err)
+               fmt.Println("Couldn't fetch active alarm list due to error: ", err)
                return alarms
        }
 
@@ -155,7 +188,7 @@ func getAlarms(flags map[string]commando.FlagValue, action alarm.AlarmAction) (a
 
        body, err := ioutil.ReadAll(resp.Body)
        if err != nil {
-               fmt.Println("ioutil.ReadAll failed: %v", err)
+               fmt.Println("ioutil.ReadAll failed: ", err)
                return alarms
        }
 
@@ -163,49 +196,58 @@ func getAlarms(flags map[string]commando.FlagValue, action alarm.AlarmAction) (a
        return alarms
 }
 
-func postAlarm(flags map[string]commando.FlagValue, a alarm.Alarm, action alarm.AlarmAction) {
-
-       // Check the interface to be used for raise or clear the alarm
-       rmr_or_http, _ := flags["if"].GetString()
-       if rmr_or_http == "rmr" {
-               alarmClient := NewAlarmClient("my-pod", "my-app")
-               if alarmClient == nil {
-                       return
-               }
-
-               // Wait until RMR is up-and-running
-               for !alarmClient.alarmer.IsRMRReady() {
-                       time.Sleep(100 * time.Millisecond)
-               }
+func postAlarmWithRmrIf(a alarm.Alarm, action alarm.AlarmAction, alarmClient *AlarmClient) {
+       if alarmClient == nil {
+               alarmClient = NewAlarmClient("my-pod", "my-app")
+       }
+       if alarmClient == nil {
+               return
+       }
 
-               if action == alarm.AlarmActionRaise {
-                       alarmClient.alarmer.Raise(a)
-               }
+       // Wait until RMR is up-and-running
+       for !alarmClient.alarmer.IsRMRReady() {
+               time.Sleep(100 * time.Millisecond)
+       }
 
-               if action == alarm.AlarmActionClear {
-                       alarmClient.alarmer.Clear(a)
-               }
-               return
+       if action == alarm.AlarmActionRaise {
+               alarmClient.alarmer.Raise(a)
        }
 
-       host, _ := flags["host"].GetString()
-       port, _ := flags["port"].GetString()
-       targetUrl := fmt.Sprintf("http://%s:%s/ric/v1/alarms", host, port)
+       if action == alarm.AlarmActionClear {
+               alarmClient.alarmer.Clear(a)
+       }
+       return
+}
 
+func postAlarmWithHttpIf(targetUrl string, a alarm.Alarm, action alarm.AlarmAction) {
        m := alarm.AlarmMessage{Alarm: a, AlarmAction: action}
        jsonData, err := json.Marshal(m)
        if err != nil {
-               fmt.Println("json.Marshal failed: %v", err)
+               fmt.Println("json.Marshal failed: ", err)
                return
        }
 
        resp, err := http.Post(targetUrl, "application/json", bytes.NewBuffer(jsonData))
        if err != nil || resp == nil {
-               fmt.Println("Couldn't fetch active alarm list due to error: %v", err)
+               fmt.Println("Couldn't fetch active alarm list due to error: ", err)
                return
        }
 }
 
+func postAlarm(flags map[string]commando.FlagValue, a alarm.Alarm, action alarm.AlarmAction, alarmClient *AlarmClient) {
+       // Check the interface to be used for raise or clear the alarm
+       rmr_or_http, _ := flags["if"].GetString()
+       if rmr_or_http == "rmr" {
+               postAlarmWithRmrIf(a, action, alarmClient)
+       } else {
+
+               host, _ := flags["host"].GetString()
+               port, _ := flags["port"].GetString()
+               targetUrl := fmt.Sprintf("http://%s:%s/ric/v1/alarms", host, port)
+               postAlarmWithHttpIf(targetUrl, a, action)
+       }
+}
+
 func displayAlarms(alarms []alarm.AlarmMessage, isHistory bool) {
        t := table.NewWriter()
        t.SetOutputMirror(os.Stdout)
@@ -242,13 +284,13 @@ func postAlarmConfig(flags map[string]commando.FlagValue) {
        m := alarm.AlarmConfigParams{MaxActiveAlarms: maxactivealarms, MaxAlarmHistory: maxalarmhistory}
        jsonData, err := json.Marshal(m)
        if err != nil {
-               fmt.Println("json.Marshal failed: %v", err)
+               fmt.Println("json.Marshal failed: ", err)
                return
        }
 
        resp, err := http.Post(targetUrl, "application/json", bytes.NewBuffer(jsonData))
        if err != nil || resp == nil {
-               fmt.Println("Couldn't fetch post alarm configuration due to error: %v", err)
+               fmt.Println("Couldn't fetch post alarm configuration due to error: ", err)
                return
        }
 }
@@ -271,13 +313,13 @@ func postAlarmDefinition(flags map[string]commando.FlagValue) {
        m := CliAlarmDefinitions{AlarmDefinitions: []*alarm.AlarmDefinition{&alarmdefinition}}
        jsonData, err := json.Marshal(m)
        if err != nil {
-               fmt.Println("json.Marshal failed: %v", err)
+               fmt.Println("json.Marshal failed: ", err)
                return
        }
 
        resp, err := http.Post(targetUrl, "application/json", bytes.NewBuffer(jsonData))
        if err != nil || resp == nil {
-               fmt.Println("Couldn't post alarm definition due to error: %v", err)
+               fmt.Println("Couldn't post alarm definition due to error: ", err)
                return
        }
 }
@@ -292,12 +334,12 @@ func deleteAlarmDefinition(flags map[string]commando.FlagValue) {
        client := &http.Client{}
        req, err := http.NewRequest("DELETE", targetUrl, nil)
        if err != nil || req == nil {
-               fmt.Println("Couldn't make delete request due to error: %v", err)
+               fmt.Println("Couldn't make delete request due to error: ", err)
                return
        }
        resp, errr := client.Do(req)
        if errr != nil || resp == nil {
-               fmt.Println("Couldn't send delete request due to error: %v", err)
+               fmt.Println("Couldn't send delete request due to error: ", err)
                return
        }
 }
@@ -313,3 +355,209 @@ func NewAlarmClient(moId, appId string) *AlarmClient {
        fmt.Println("Failed to create alarmInstance", err)
        return nil
 }
+
+// Conduct performance testing
+func conductperformancetest(flags map[string]commando.FlagValue) {
+       var readerror error
+       var senderror error
+       var readobjerror error
+       host, _ := flags["host"].GetString()
+       port, _ := flags["port"].GetString()
+       targetUrl := fmt.Sprintf("http://%s:%s/ric/v1/alarms/define", host, port)
+       readerror = readPerfAlarmDefinitionFromJson()
+       if readerror == nil {
+               senderror = sendPerfAlarmDefinitionToAlarmManager(targetUrl)
+               if senderror == nil {
+                       fmt.Println("sent performance alarm definitions to alarm manager")
+                       CLIPerfAlarmObjects = make(map[int]*alarm.Alarm)
+                       readobjerror = readPerfAlarmObjectFromJson()
+                       if readobjerror == nil {
+                               profile, _ := flags["prf"].GetInt()
+                               if profile == 1 {
+                                       fmt.Println("starting peak performance test")
+                                       peakPerformanceTest(flags)
+                               } else if profile == 2 {
+                                       fmt.Println("starting endurance test")
+                                       enduranceTest(flags)
+                               } else {
+                                       fmt.Println("Unknown profile, received profile = ", profile)
+                               }
+                       } else {
+                               fmt.Println("reading performance alarm objects from json file failed ")
+                       }
+               } else {
+                       fmt.Println("sending performance alarm definitions to alarm manager failed ")
+               }
+
+       } else {
+               fmt.Println("reading performance alarm definitions from json file failed ")
+       }
+
+}
+
+func peakPerformanceTest(flags map[string]commando.FlagValue) {
+       nalarms, _ := flags["nal"].GetInt()
+       var count int = 0
+       for aid, obj := range CLIPerfAlarmObjects {
+               count = count + 1
+               if count <= nalarms {
+                       fmt.Println("peakPerformanceTest: invoking worker routine ", count, aid, *obj)
+                       wg.Add(1)
+                       go raiseClearAlarmOnce(obj, flags)
+               } else {
+                       break
+               }
+       }
+       fmt.Println("peakPerformanceTest: Waiting for workers to finish")
+       wg.Wait()
+       fmt.Println("peakPerformanceTest: Wait completed")
+}
+
+func enduranceTest(flags map[string]commando.FlagValue) {
+       alarmspersec, _ := flags["aps"].GetInt()
+       var count int = 0
+       for aid, obj := range CLIPerfAlarmObjects {
+               count = count + 1
+               if count <= alarmspersec {
+                       fmt.Println("enduranceTest: invoking worker routine ", count, aid, *obj)
+                       wg.Add(1)
+                       go raiseClearAlarmOverPeriod(obj, flags)
+               } else {
+                       break
+               }
+       }
+       fmt.Println("enduranceTest: Waiting for workers to finish")
+       wg.Wait()
+       fmt.Println("enduranceTest: Wait completed")
+}
+
+func readPerfAlarmObjectFromJson() error {
+       filename := os.Getenv("PERF_OBJ_FILE")
+       file, err := ioutil.ReadFile(filename)
+       if err == nil {
+               data := RicPerfAlarmObjects{}
+               err = json.Unmarshal([]byte(file), &data)
+               if err == nil {
+                       for _, alarmObject := range data.AlarmObjects {
+                               ricAlarmObject := new(alarm.Alarm)
+                               ricAlarmObject.ManagedObjectId = alarmObject.ManagedObjectId
+                               ricAlarmObject.ApplicationId = alarmObject.ApplicationId
+                               ricAlarmObject.SpecificProblem = alarmObject.SpecificProblem
+                               ricAlarmObject.PerceivedSeverity = alarmObject.PerceivedSeverity
+                               ricAlarmObject.AdditionalInfo = alarmObject.AdditionalInfo
+                               ricAlarmObject.IdentifyingInfo = alarmObject.IdentifyingInfo
+                               CLIPerfAlarmObjects[alarmObject.SpecificProblem] = ricAlarmObject
+                       }
+               } else {
+                       fmt.Println("readPerfAlarmObjectFromJson: json.Unmarshal failed with error ", err)
+                       return err
+               }
+       } else {
+               fmt.Println("readPerfAlarmObjectFromJson: ioutil.ReadFile failed with error ", err)
+               return err
+       }
+       return nil
+}
+
+func readPerfAlarmDefinitionFromJson() error {
+       filename := os.Getenv("PERF_DEF_FILE")
+       file, err := ioutil.ReadFile(filename)
+       if err == nil {
+               data := CliAlarmDefinitions{}
+               err = json.Unmarshal([]byte(file), &data)
+               if err == nil {
+                       for _, alarmDefinition := range data.AlarmDefinitions {
+                               _, exists := alarm.RICAlarmDefinitions[alarmDefinition.AlarmId]
+                               if exists {
+                                       fmt.Println("ReadPerfAlarmDefinitionFromJson: alarm definition already exists for ", alarmDefinition.AlarmId)
+                               } else {
+                                       fmt.Println("ReadPerfAlarmDefinitionFromJson: alarm ", alarmDefinition.AlarmId)
+                                       ricAlarmDefintion := new(alarm.AlarmDefinition)
+                                       ricAlarmDefintion.AlarmId = alarmDefinition.AlarmId
+                                       ricAlarmDefintion.AlarmText = alarmDefinition.AlarmText
+                                       ricAlarmDefintion.EventType = alarmDefinition.EventType
+                                       ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions
+                                       CliPerfAlarmDefinitions.AlarmDefinitions = append(CliPerfAlarmDefinitions.AlarmDefinitions, ricAlarmDefintion)
+                               }
+                       }
+               } else {
+                       fmt.Println("ReadPerfAlarmDefinitionFromJson: json.Unmarshal failed with error: ", err)
+                       return err
+               }
+       } else {
+               fmt.Println("ReadPerfAlarmDefinitionFromJson: ioutil.ReadFile failed with error: ", err)
+               return err
+       }
+       return nil
+}
+
+func sendPerfAlarmDefinitionToAlarmManager(targetUrl string) error {
+
+       jsonData, err := json.Marshal(CliPerfAlarmDefinitions)
+       if err != nil {
+               fmt.Println("sendPerfAlarmDefinitionToAlarmManager: json.Marshal failed: ", err)
+               return err
+       }
+
+       resp, err := http.Post(targetUrl, "application/json", bytes.NewBuffer(jsonData))
+       if err != nil || resp == nil {
+               fmt.Println("sendPerfAlarmDefinitionToAlarmManager: Couldn't post alarm definition to targeturl due to error: ", targetUrl, err)
+               return err
+       }
+       return nil
+}
+
+func wakeUpAfterTime(timeinseconds int, chn chan string, action string) {
+       time.Sleep(time.Second * time.Duration(timeinseconds))
+       chn <- action
+}
+
+func raiseClearAlarmOnce(alarmobject *alarm.Alarm, flags map[string]commando.FlagValue) {
+       var alarmClient *AlarmClient = nil
+       defer wg.Done()
+       chn := make(chan string, 1)
+       rmr_or_http, _ := flags["if"].GetString()
+       if rmr_or_http == "rmr" {
+               alarmClient = NewAlarmClient("my-pod", "my-app")
+       }
+       postAlarm(flags, *alarmobject, alarm.AlarmActionRaise, alarmClient)
+       go wakeUpAfterTime(PeakTestDuration, chn, Clear)
+       select {
+       case res := <-chn:
+               if res == Clear {
+                       postAlarm(flags, *alarmobject, alarm.AlarmActionClear, alarmClient)
+                       go wakeUpAfterTime(PeakTestDuration, chn, End)
+               } else if res == End {
+                       return
+               }
+       }
+}
+
+func raiseClearAlarmOverPeriod(alarmobject *alarm.Alarm, flags map[string]commando.FlagValue) {
+       var alarmClient *AlarmClient = nil
+       defer wg.Done()
+       timeinminutes, _ := flags["tim"].GetInt()
+       timeinseconds := timeinminutes * 60
+       chn := make(chan string, 1)
+       rmr_or_http, _ := flags["if"].GetString()
+       if rmr_or_http == "rmr" {
+               alarmClient = NewAlarmClient("my-pod", "my-app")
+       }
+       postAlarm(flags, *alarmobject, alarm.AlarmActionRaise, alarmClient)
+       go wakeUpAfterTime(OneSecondDuration, chn, Clear)
+       go wakeUpAfterTime(timeinseconds, chn, End)
+       for {
+               select {
+               case res := <-chn:
+                       if res == Raise {
+                               postAlarm(flags, *alarmobject, alarm.AlarmActionRaise, alarmClient)
+                               go wakeUpAfterTime(OneSecondDuration, chn, Clear)
+                       } else if res == Clear {
+                               postAlarm(flags, *alarmobject, alarm.AlarmActionClear, alarmClient)
+                               go wakeUpAfterTime(OneSecondDuration, chn, Raise)
+                       } else if res == End {
+                               return
+                       }
+               }
+       }
+}
diff --git a/go.mod b/go.mod
index b6ae937..0fb43fa 100644 (file)
--- a/go.mod
+++ b/go.mod
@@ -20,6 +20,8 @@ require (
        github.com/gorilla/mux v1.7.1
        github.com/jedib0t/go-pretty v4.3.0+incompatible
        github.com/mattn/go-runewidth v0.0.9 // indirect
+       github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+       github.com/modern-go/reflect2 v1.0.1 // indirect
        github.com/prometheus/alertmanager v0.20.0
        github.com/spf13/viper v1.6.2
        github.com/stretchr/testify v1.5.1
index 0cb528f..f00a812 100755 (executable)
@@ -75,8 +75,10 @@ func (a *AlarmManager) HandleAlarms(rp *app.RMRParams) (*alert.PostAlertsOK, err
 }
 
 func (a *AlarmManager) ProcessAlarm(m *alarm.AlarmMessage) (*alert.PostAlertsOK, error) {
+       a.mutex.Lock()
        if _, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok {
                app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem)
+               a.mutex.Unlock()
                return nil, nil
        }
 
@@ -86,6 +88,7 @@ func (a *AlarmManager) ProcessAlarm(m *alarm.AlarmMessage) (*alert.PostAlertsOK,
                app.Logger.Info("Duplicate alarm found, suppressing ...")
                if m.PerceivedSeverity == a.activeAlarms[idx].PerceivedSeverity {
                        // Duplicate with same severity found
+                       a.mutex.Unlock()
                        return nil, nil
                } else {
                        // Remove duplicate with different severity
@@ -98,27 +101,37 @@ func (a *AlarmManager) ProcessAlarm(m *alarm.AlarmMessage) (*alert.PostAlertsOK,
                if found {
                        a.alarmHistory = append(a.alarmHistory, *m)
                        a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
-                       if len(a.alarmHistory) >= a.maxAlarmHistory {
+                       if ((len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false)){
                                app.Logger.Error("alarm history count exceeded maxAlarmHistory threshold")
                                histAlarm := a.alarmClient.NewAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, alarm.SeverityWarning, "threshold", "history")
                                histAlarmMessage := alarm.AlarmMessage{Alarm: histAlarm, AlarmAction: alarm.AlarmActionRaise, AlarmTime: (time.Now().UnixNano())}
                                a.activeAlarms = append(a.activeAlarms, histAlarmMessage)
                                a.alarmHistory = append(a.alarmHistory, histAlarmMessage)
                        }
+                       if ((a.exceededActiveAlarmOn == true) && (m.Alarm.SpecificProblem == alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD)) {
+                               a.exceededActiveAlarmOn = false
+                       }
+                       if ((a.exceededAlarmHistoryOn == true) && (m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD)) {
+                               a.exceededAlarmHistoryOn = false
+                       }
                        if a.postClear {
+                               a.mutex.Unlock()
                                return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusResolved, m.AlarmTime))
                        }
                }
                app.Logger.Info("No matching active alarm found, suppressing ...")
+               a.mutex.Unlock()
                return nil, nil
        }
 
        // New alarm -> update active alarms and post to Alert Manager
        if m.AlarmAction == alarm.AlarmActionRaise {
                a.UpdateAlarmLists(m)
+               a.mutex.Unlock()
                return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime))
        }
 
+       a.mutex.Unlock()
        return nil, nil
 }
 
@@ -133,34 +146,30 @@ func (a *AlarmManager) IsMatchFound(newAlarm alarm.Alarm) (int, bool) {
 }
 
 func (a *AlarmManager) RemoveAlarm(alarms []alarm.AlarmMessage, i int, listName string) []alarm.AlarmMessage {
-       a.mutex.Lock()
-       defer a.mutex.Unlock()
-
        app.Logger.Info("Alarm '%+v' deleted from the '%s' list", alarms[i], listName)
        copy(alarms[i:], alarms[i+1:])
        return alarms[:len(alarms)-1]
 }
 
 func (a *AlarmManager) UpdateAlarmLists(newAlarm *alarm.AlarmMessage) {
-       a.mutex.Lock()
-       defer a.mutex.Unlock()
-
        /* If maximum number of active alarms is reached, an error log writing is made, and new alarm indicating the problem is raised.
           The attempt to raise the alarm next time will be supressed when found as duplicate. */
-       if len(a.activeAlarms) >= a.maxActiveAlarms {
+       if ((len(a.activeAlarms) >= a.maxActiveAlarms) && (a.exceededActiveAlarmOn == false)) {
                app.Logger.Error("active alarm count exceeded maxActiveAlarms threshold")
                actAlarm := a.alarmClient.NewAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, alarm.SeverityWarning, "threshold", "active")
                actAlarmMessage := alarm.AlarmMessage{Alarm: actAlarm, AlarmAction: alarm.AlarmActionRaise, AlarmTime: (time.Now().UnixNano())}
                a.activeAlarms = append(a.activeAlarms, actAlarmMessage)
                a.alarmHistory = append(a.alarmHistory, actAlarmMessage)
+               a.exceededActiveAlarmOn = true
        }
 
-       if len(a.alarmHistory) >= a.maxAlarmHistory {
+       if ((len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false)) {
                app.Logger.Error("alarm history count exceeded maxAlarmHistory threshold")
                histAlarm := a.alarmClient.NewAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, alarm.SeverityWarning, "threshold", "history")
                histAlarmMessage := alarm.AlarmMessage{Alarm: histAlarm, AlarmAction: alarm.AlarmActionRaise, AlarmTime: (time.Now().UnixNano())}
                a.activeAlarms = append(a.activeAlarms, histAlarmMessage)
                a.alarmHistory = append(a.alarmHistory, histAlarmMessage)
+               a.exceededAlarmHistoryOn = true
        }
 
        // @todo: For now just keep the alarms (both active and history) in-memory. Use SDL later for persistence
@@ -258,10 +267,10 @@ func (a *AlarmManager) ReadAlarmDefinitionFromJson() {
                                }
                        }
                } else {
-                       app.Logger.Error("json.Unmarshal failed with error %v", err)
+                       app.Logger.Error("ReadAlarmDefinitionFromJson: json.Unmarshal failed with error %v", err)
                }
        } else {
-               app.Logger.Error("ioutil.ReadFile failed with error %v", err)
+               app.Logger.Error("ReadAlarmDefinitionFromJson: ioutil.ReadFile failed with error %v", err)
        }
 }
 
@@ -312,6 +321,8 @@ func NewAlarmManager(amHost string, alertInterval int) *AlarmManager {
                alarmHistory:    make([]alarm.AlarmMessage, 0),
                maxActiveAlarms: app.Config.GetInt("controls.maxActiveAlarms"),
                maxAlarmHistory: app.Config.GetInt("controls.maxAlarmHistory"),
+               exceededActiveAlarmOn:  false,
+               exceededAlarmHistoryOn: false,
        }
 }
 
index f4a5aee..34b3902 100755 (executable)
@@ -48,6 +48,7 @@ var eventChan chan string
 // Test cases
 func TestMain(M *testing.M) {
        alarmManager = NewAlarmManager("localhost:9093", 500)
+       alarmManager.alertInterval = 20000
        go alarmManager.Run(false)
        time.Sleep(time.Duration(10) * time.Second)
 
@@ -238,6 +239,7 @@ func TestMultipleAlarmsRaisedSucess(t *testing.T) {
        b := alarmer.NewAlarm(alarm.TCP_CONNECTIVITY_LOST_TO_DBAAS, alarm.SeverityMinor, "Hello", "abcd 11")
        assert.Nil(t, alarmer.Raise(b), "raise failed")
 
+       time.Sleep(time.Duration(2) * time.Second)
        VerifyAlarm(t, a, 2)
        VerifyAlarm(t, b, 2)
 }
@@ -332,7 +334,7 @@ func TestActiveAlarmMaxThresholds(t *testing.T) {
 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
        receivedAlert := waitForEvent()
 
-       assert.Equal(t, len(alarmManager.activeAlarms), expectedCount)
+       assert.Equal(t, expectedCount, len(alarmManager.activeAlarms))
        _, ok := alarmManager.IsMatchFound(a)
        assert.True(t, ok)
 
index 7a09083..80f4997 100755 (executable)
@@ -96,6 +96,7 @@ func (a *AlarmManager) SetAlarmDefinition(w http.ResponseWriter, r *http.Request
                        ricAlarmDefintion.EventType = alarmDefinition.EventType
                        ricAlarmDefintion.OperationInstructions = alarmDefinition.OperationInstructions
                        alarm.RICAlarmDefinitions[alarmDefinition.AlarmId] = ricAlarmDefintion
+                       app.Logger.Debug("POST - alarm definition added for alarm id %v", alarmDefinition.AlarmId)
                }
        }
 
index 73daf48..c415fda 100755 (executable)
@@ -39,6 +39,8 @@ type AlarmManager struct {
        maxActiveAlarms int
        maxAlarmHistory int
        alarmClient     *alarm.RICAlarm
+       exceededActiveAlarmOn bool
+       exceededAlarmHistoryOn bool
 }
 
 type AlertStatus string
@@ -54,3 +56,7 @@ var Hash string
 type RicAlarmDefinitions struct {
        AlarmDefinitions []*alarm.AlarmDefinition `json:"alarmdefinitions"`
 }
+
+type RicPerfAlarmObjects struct {
+       AlarmObjects []*alarm.Alarm `json:"alarmobjects"`
+}
diff --git a/perfresources/perf-alarm-definition.json b/perfresources/perf-alarm-definition.json
new file mode 100755 (executable)
index 0000000..e12fb0f
--- /dev/null
@@ -0,0 +1,304 @@
+{
+    "alarmdefinitions" : [
+        {
+            "alarmId" : 1001,
+            "alarmText" : "PERFORMANCE TEST ALARM 1",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1002,
+            "alarmText" : "PERFORMANCE TEST ALARM 2",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1003,
+            "alarmText" : "PERFORMANCE TEST ALARM 3",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1004,
+            "alarmText" : "PERFORMANCE TEST ALARM 4",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1005,
+            "alarmText" : "PERFORMANCE TEST ALARM 5",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1006,
+            "alarmText" : "PERFORMANCE TEST ALARM 6",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1007,
+            "alarmText" : "PERFORMANCE TEST ALARM 7",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1008,
+            "alarmText" : "PERFORMANCE TEST ALARM 8",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1009,
+            "alarmText" : "PERFORMANCE TEST ALARM 9",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1010,
+            "alarmText" : "PERFORMANCE TEST ALARM 10",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1011,
+            "alarmText" : "PERFORMANCE TEST ALARM 11",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1012,
+            "alarmText" : "PERFORMANCE TEST ALARM 12",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1013,
+            "alarmText" : "PERFORMANCE TEST ALARM 13",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1014,
+            "alarmText" : "PERFORMANCE TEST ALARM 14",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1015,
+            "alarmText" : "PERFORMANCE TEST ALARM 15",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1016,
+            "alarmText" : "PERFORMANCE TEST ALARM 16",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1017,
+            "alarmText" : "PERFORMANCE TEST ALARM 17",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1018,
+            "alarmText" : "PERFORMANCE TEST ALARM 18",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1019,
+            "alarmText" : "PERFORMANCE TEST ALARM 19",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1020,
+            "alarmText" : "PERFORMANCE TEST ALARM 20",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1021,
+            "alarmText" : "PERFORMANCE TEST ALARM 21",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1022,
+            "alarmText" : "PERFORMANCE TEST ALARM 22",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1023,
+            "alarmText" : "PERFORMANCE TEST ALARM 23",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1024,
+            "alarmText" : "PERFORMANCE TEST ALARM 24",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1025,
+            "alarmText" : "PERFORMANCE TEST ALARM 25",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1026,
+            "alarmText" : "PERFORMANCE TEST ALARM 26",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1027,
+            "alarmText" : "PERFORMANCE TEST ALARM 27",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1028,
+            "alarmText" : "PERFORMANCE TEST ALARM 28",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1029,
+            "alarmText" : "PERFORMANCE TEST ALARM 29",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1030,
+            "alarmText" : "PERFORMANCE TEST ALARM 30",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1031,
+            "alarmText" : "PERFORMANCE TEST ALARM 31",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1032,
+            "alarmText" : "PERFORMANCE TEST ALARM 32",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1033,
+            "alarmText" : "PERFORMANCE TEST ALARM 33",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1034,
+            "alarmText" : "PERFORMANCE TEST ALARM 34",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1035,
+            "alarmText" : "PERFORMANCE TEST ALARM 35",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1036,
+            "alarmText" : "PERFORMANCE TEST ALARM 36",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1037,
+            "alarmText" : "PERFORMANCE TEST ALARM 37",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1038,
+            "alarmText" : "PERFORMANCE TEST ALARM 38",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1039,
+            "alarmText" : "PERFORMANCE TEST ALARM 39",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1040,
+            "alarmText" : "PERFORMANCE TEST ALARM 40",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1041,
+            "alarmText" : "PERFORMANCE TEST ALARM 41",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1042,
+            "alarmText" : "PERFORMANCE TEST ALARM 42",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1043,
+            "alarmText" : "PERFORMANCE TEST ALARM 43",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1044,
+            "alarmText" : "PERFORMANCE TEST ALARM 44",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1045,
+            "alarmText" : "PERFORMANCE TEST ALARM 45",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1046,
+            "alarmText" : "PERFORMANCE TEST ALARM 46",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1047,
+            "alarmText" : "PERFORMANCE TEST ALARM 47",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1048,
+            "alarmText" : "PERFORMANCE TEST ALARM 48",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1049,
+            "alarmText" : "PERFORMANCE TEST ALARM 49",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        },
+        {
+            "alarmId" : 1050,
+            "alarmText" : "PERFORMANCE TEST ALARM 50",
+            "eventType" : "Performance test",
+            "operationInstructions" : "Not defined"
+        }
+   ]
+}
diff --git a/perfresources/perf-alarm-object.json b/perfresources/perf-alarm-object.json
new file mode 100755 (executable)
index 0000000..0c6734a
--- /dev/null
@@ -0,0 +1,404 @@
+{
+    "alarmobjects" : [
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP1",
+            "specificProblem" : 1001,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP2",
+            "specificProblem" : 1002,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP3",
+            "specificProblem" : 1003,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP4",
+            "specificProblem" : 1004,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP5",
+            "specificProblem" : 1005,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP6",
+            "specificProblem" : 1006,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP7",
+            "specificProblem" : 1007,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP8",
+            "specificProblem" : 1008,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP9",
+            "specificProblem" : 1009,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP10",
+            "specificProblem" : 1010,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP11",
+            "specificProblem" : 1011,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP12",
+            "specificProblem" : 1012,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP13",
+            "specificProblem" : 1013,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP14",
+            "specificProblem" : 1014,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP15",
+            "specificProblem" : 1015,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP16",
+            "specificProblem" : 1016,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP17",
+            "specificProblem" : 1017,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP18",
+            "specificProblem" : 1018,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP19",
+            "specificProblem" : 1019,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP20",
+            "specificProblem" : 1020,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP21",
+            "specificProblem" : 1021,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP22",
+            "specificProblem" : 1022,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP23",
+            "specificProblem" : 1023,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP24",
+            "specificProblem" : 1024,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP25",
+            "specificProblem" : 1025,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP26",
+            "specificProblem" : 1026,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP27",
+            "specificProblem" : 1027,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP28",
+            "specificProblem" : 1028,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP29",
+            "specificProblem" : 1029,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP30",
+            "specificProblem" : 1030,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP31",
+            "specificProblem" : 1031,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP32",
+            "specificProblem" : 1032,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP33",
+            "specificProblem" : 1033,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP34",
+            "specificProblem" : 1034,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP35",
+            "specificProblem" : 1035,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP36",
+            "specificProblem" : 1036,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP37",
+            "specificProblem" : 1037,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP38",
+            "specificProblem" : 1038,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP39",
+            "specificProblem" : 1039,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP40",
+            "specificProblem" : 1040,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP41",
+            "specificProblem" : 1041,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP42",
+            "specificProblem" : 1042,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP43",
+            "specificProblem" : 1043,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP44",
+            "specificProblem" : 1044,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP45",
+            "specificProblem" : 1045,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP46",
+            "specificProblem" : 1046,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP47",
+            "specificProblem" : 1047,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP48",
+            "specificProblem" : 1048,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP49",
+            "specificProblem" : 1049,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        },
+        {
+            "managedObjectId" : "SEP",
+            "applicationId" : "PERFAPP50",
+            "specificProblem" : 1050,
+            "perceivedSeverity" : "WARNING",
+            "additionalInfo" : "capacity",
+            "identifyingInfo" : "performance"
+        }
+   ]
+}