"github.com/jedib0t/go-pretty/table"
"github.com/prometheus/alertmanager/api/v2/client"
"github.com/prometheus/alertmanager/api/v2/client/alert"
- "github.com/thatisuday/commando"
"github.com/spf13/viper"
+ "github.com/thatisuday/commando"
)
type CliAlarmDefinitions struct {
AlarmObjects []*alarm.Alarm `json:"alarmobjects"`
}
+type AlarmNotification struct {
+ alarm.AlarmMessage
+ alarm.AlarmDefinition
+}
+
var CLIPerfAlarmObjects map[int]*alarm.Alarm
var wg sync.WaitGroup
)
func main() {
+ alarmManagerHost := "localhost"
+ if h := os.Getenv("ALARM_MANAGER_HOST"); h != "" {
+ alarmManagerHost = h
+ }
+
+ alertManagerHost := "localhost"
+ if h := os.Getenv("ALERT_MANAGER_HOST"); h != "" {
+ alertManagerHost = h
+ }
// configure commando
commando.
Register("active").
SetShortDescription("Displays the SEP active alarms").
SetDescription("This command displays more information about the SEP active alarms").
- AddFlag("host", "Alarm manager host address", commando.String, "localhost").
+ AddFlag("host", "Alarm manager host address", commando.String, alarmManagerHost).
AddFlag("port", "Alarm manager host address", commando.String, "8080").
SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) {
displayAlarms(getAlarms(flags, "active"), false)
Register("history").
SetShortDescription("Displays the SEP alarm history").
SetDescription("This command displays more information about the SEP alarm history").
- AddFlag("host", "Alarm manager host address", commando.String, "localhost").
+ AddFlag("host", "Alarm manager host address", commando.String, alarmManagerHost).
AddFlag("port", "Alarm manager host address", commando.String, "8080").
SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) {
displayAlarms(getAlarms(flags, "history"), true)
AddFlag("severity", "Perceived severity", commando.String, nil).
AddFlag("iinfo", "Application identifying info", commando.String, nil).
AddFlag("aai", "Application additional info", commando.String, "-").
- AddFlag("host", "Alarm manager host address", commando.String, "localhost").
+ AddFlag("host", "Alarm manager host address", commando.String, alarmManagerHost).
AddFlag("port", "Alarm manager host address", commando.String, "8080").
AddFlag("if", "http or rmr used as interface", commando.String, "http").
SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) {
AddFlag("apid", "Application Id", commando.String, nil).
AddFlag("sp", "Specific problem Id", commando.Int, nil).
AddFlag("iinfo", "Application identifying info", commando.String, nil).
- AddFlag("host", "Alarm manager host address", commando.String, "localhost").
+ AddFlag("host", "Alarm manager host address", commando.String, alarmManagerHost).
AddFlag("port", "Alarm manager host address", commando.String, "8080").
AddFlag("if", "http or rmr used as interface", commando.String, "http").
SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) {
SetShortDescription("Configure alarm manager with given parameters").
AddFlag("mal", "max active alarms", commando.Int, nil).
AddFlag("mah", "max alarm history", commando.Int, nil).
- AddFlag("host", "Alarm manager host address", commando.String, "localhost").
+ AddFlag("host", "Alarm manager host address", commando.String, alarmManagerHost).
AddFlag("port", "Alarm manager host address", commando.String, "8080").
SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) {
postAlarmConfig(flags)
AddFlag("atx", "alarm text", commando.String, nil).
AddFlag("ety", "event type", commando.String, nil).
AddFlag("oin", "operation instructions", commando.String, nil).
- AddFlag("host", "Alarm manager host address", commando.String, "localhost").
+ AddFlag("host", "Alarm manager host address", commando.String, alarmManagerHost).
AddFlag("port", "Alarm manager host address", commando.String, "8080").
SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) {
postAlarmDefinition(flags)
Register("undefine").
SetShortDescription("Define alarm with given parameters").
AddFlag("aid", "alarm identifier", commando.Int, nil).
- AddFlag("host", "Alarm manager host address", commando.String, "localhost").
+ AddFlag("host", "Alarm manager host address", commando.String, alarmManagerHost).
AddFlag("port", "Alarm manager host address", commando.String, "8080").
SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) {
deleteAlarmDefinition(flags)
AddFlag("nal", "number of alarms", commando.Int, nil).
AddFlag("aps", "alarms per sec", commando.Int, nil).
AddFlag("tim", "total time of test", commando.Int, nil).
- AddFlag("host", "Alarm manager host address", commando.String, "localhost").
+ AddFlag("host", "Alarm manager host address", commando.String, alarmManagerHost).
AddFlag("port", "Alarm manager host address", commando.String, "8080").
AddFlag("if", "http or rmr used as interface", commando.String, "http").
SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) {
AddFlag("inhibited", "Inhibited alerts in Prometheus Alert Manager", commando.Bool, true).
AddFlag("silenced", "Silenced alerts in Prometheus Alert Manager", commando.Bool, true).
AddFlag("unprocessed", "Unprocessed alerts in Prometheus Alert Manager", commando.Bool, true).
- AddFlag("host", "Prometheus Alert Manager host address", commando.String, nil).
+ AddFlag("host", "Prometheus Alert Manager host address", commando.String, alertManagerHost).
AddFlag("port", "Prometheus Alert Manager port", commando.String, "9093").
SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) {
displayAlerts(flags)
return
}
-func getAlarms(flags map[string]commando.FlagValue, action alarm.AlarmAction) (alarms []alarm.AlarmMessage) {
+func getAlarms(flags map[string]commando.FlagValue, action alarm.AlarmAction) (alarms []AlarmNotification) {
host, _ := flags["host"].GetString()
port, _ := flags["port"].GetString()
targetUrl := fmt.Sprintf("http://%s:%s/ric/v1/alarms/%s", host, port, action)
targetUrl := fmt.Sprintf("http://%s:%s/ric/v1/alarms", host, port)
postAlarmWithHttpIf(targetUrl, a, action)
}
+ fmt.Println("command executed successfully!")
}
-func displayAlarms(alarms []alarm.AlarmMessage, isHistory bool) {
+func displayAlarms(alarms []AlarmNotification, isHistory bool) {
t := table.NewWriter()
t.SetOutputMirror(os.Stdout)
if isHistory {
- t.AppendHeader(table.Row{"SP", "MOID", "APPID", "IINFO", "SEVERITY", "AAI", "ACTION", "TIME"})
+ t.AppendHeader(table.Row{"ID", "SP", "MOID", "APPID", "IINFO", "SEVERITY", "AAI", "ACTION", "TIME"})
} else {
- t.AppendHeader(table.Row{"SP", "MOID", "APPID", "IINFO", "SEVERITY", "AAI", "TIME"})
+ t.AppendHeader(table.Row{"ID", "SP", "MOID", "APPID", "IINFO", "SEVERITY", "AAI", "TIME"})
}
for _, a := range alarms {
alarmTime := time.Unix(0, a.AlarmTime).Format("02/01/2006, 15:04:05")
if isHistory {
t.AppendRows([]table.Row{
- {a.SpecificProblem, a.ManagedObjectId, a.ApplicationId, a.IdentifyingInfo, a.PerceivedSeverity, a.AdditionalInfo, a.AlarmAction, alarmTime},
+ {a.AlarmId, a.SpecificProblem, a.ManagedObjectId, a.ApplicationId, a.IdentifyingInfo, a.PerceivedSeverity, a.AdditionalInfo, a.AlarmAction, alarmTime},
})
} else {
t.AppendRows([]table.Row{
- {a.SpecificProblem, a.ManagedObjectId, a.ApplicationId, a.IdentifyingInfo, a.PerceivedSeverity, a.AdditionalInfo, alarmTime},
+ {a.AlarmId, a.SpecificProblem, a.ManagedObjectId, a.ApplicationId, a.IdentifyingInfo, a.PerceivedSeverity, a.AdditionalInfo, alarmTime},
})
}
}
t := table.NewWriter()
t.SetOutputMirror(os.Stdout)
t.AppendHeader(table.Row{"Alerts from Prometheus Alert Manager"})
- for _, gettableAlert := range resp.Payload{
+ for _, gettableAlert := range resp.Payload {
t.AppendRow([]interface{}{"------------------------------------"})
if gettableAlert != nil {
for key, item := range gettableAlert.Annotations {
- t.AppendRow([]interface{}{key, item})
+ t.AppendRow([]interface{}{key, item})
}
if gettableAlert.EndsAt != nil {
t.AppendRow([]interface{}{"EndsAt", *gettableAlert.EndsAt})
}
for key, item := range gettableAlert.Receivers {
if gettableAlert.Receivers != nil {
- t.AppendRow([]interface{}{key, *item.Name})
+ t.AppendRow([]interface{}{key, *item.Name})
}
}
if gettableAlert.StartsAt != nil {
}
t.AppendRow([]interface{}{"GeneratorURL", gettableAlert.Alert.GeneratorURL})
for key, item := range gettableAlert.Alert.Labels {
- t.AppendRow([]interface{}{key, item})
+ t.AppendRow([]interface{}{key, item})
}
}
}
t.SetStyle(table.StyleColoredBright)
t.Render()
}
-
+
func getAlerts(flags map[string]commando.FlagValue) (*alert.GetAlertsOK, error) {
active, _ := flags["active"].GetBool()
inhibited, _ := flags["inhibited"].GetBool()
cr := clientruntime.New(amAddress, amBaseUrl, amSchemes)
return client.New(cr, strfmt.Default)
}
-
package main
import (
+ "bytes"
"encoding/json"
"fmt"
"gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
"github.com/prometheus/alertmanager/api/v2/models"
"github.com/spf13/viper"
"io/ioutil"
+ "net/http"
"os"
"time"
)
}
app.Logger.Info("newAlarm: %v", m)
- return a.ProcessAlarm(&AlarmInformation{m, alarm.AlarmDefinition{}})
+ return a.ProcessAlarm(&AlarmNotification{m, alarm.AlarmDefinition{}})
}
-func (a *AlarmManager) ProcessAlarm(m *AlarmInformation) (*alert.PostAlertsOK, error) {
+func (a *AlarmManager) ProcessAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
a.mutex.Lock()
if _, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok {
app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem)
// Clear alarm if found from active alarm list
if m.AlarmAction == alarm.AlarmActionClear {
if found {
+ a.UpdateAlarmFields(a.activeAlarms[idx].AlarmId, m)
a.alarmHistory = append(a.alarmHistory, *m)
a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
- app.Logger.Error("alarm history count exceeded maxAlarmHistory threshold")
- histAlarm := a.alarmClient.NewAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, alarm.SeverityWarning, "threshold", "history")
- am := alarm.AlarmMessage{Alarm: histAlarm, AlarmAction: alarm.AlarmActionRaise, AlarmTime: (time.Now().UnixNano())}
- histAlarmMessage := AlarmInformation{am, alarm.AlarmDefinition{}}
- a.activeAlarms = append(a.activeAlarms, histAlarmMessage)
- a.alarmHistory = append(a.alarmHistory, histAlarmMessage)
+ app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
+ a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
}
- if (a.exceededActiveAlarmOn == true) && (m.Alarm.SpecificProblem == alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD) {
+
+ if a.exceededActiveAlarmOn && m.Alarm.SpecificProblem == alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD {
a.exceededActiveAlarmOn = false
}
- if (a.exceededAlarmHistoryOn == true) && (m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD) {
+
+ if a.exceededAlarmHistoryOn && m.Alarm.SpecificProblem == alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD {
a.exceededAlarmHistoryOn = false
}
+
if a.postClear {
a.mutex.Unlock()
+
+ // Send alarm notification to NOMA, if enabled
+ if app.Config.GetBool("controls.noma.enabled") {
+ m.PerceivedSeverity = alarm.SeverityCleared
+ return a.PostAlarm(m)
+ }
return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusResolved, m.AlarmTime))
}
}
// New alarm -> update active alarms and post to Alert Manager
if m.AlarmAction == alarm.AlarmActionRaise {
+ a.UpdateAlarmFields(a.GenerateAlarmId(), m)
a.UpdateAlarmLists(m)
a.mutex.Unlock()
+
+ // Send alarm notification to NOMA, if enabled
+ if app.Config.GetBool("controls.noma.enabled") {
+ return a.PostAlarm(m)
+ }
return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime))
}
return -1, false
}
-func (a *AlarmManager) RemoveAlarm(alarms []AlarmInformation, i int, listName string) []AlarmInformation {
+func (a *AlarmManager) RemoveAlarm(alarms []AlarmNotification, i int, listName string) []AlarmNotification {
app.Logger.Info("Alarm '%+v' deleted from the '%s' list", alarms[i], listName)
copy(alarms[i:], alarms[i+1:])
return alarms[:len(alarms)-1]
}
-func (a *AlarmManager) UpdateAlarmFields(newAlarm *AlarmInformation) {
- alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
- newAlarm.AlarmId = a.uniqueAlarmId
+func (a *AlarmManager) GenerateAlarmId() int {
a.uniqueAlarmId++ // @todo: generate a unique ID
+ return a.uniqueAlarmId
+}
+
+func (a *AlarmManager) UpdateAlarmFields(alarmId int, newAlarm *AlarmNotification) {
+ alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
+ newAlarm.AlarmId = alarmId
newAlarm.AlarmText = alarmDef.AlarmText
newAlarm.EventType = alarmDef.EventType
}
-func (a *AlarmManager) UpdateAlarmLists(newAlarm *AlarmInformation) {
+func (a *AlarmManager) GenerateThresholdAlarm(sp int, data string) bool {
+ thresholdAlarm := a.alarmClient.NewAlarm(sp, alarm.SeverityWarning, "threshold", data)
+ thresholdMessage := alarm.AlarmMessage{
+ Alarm: thresholdAlarm,
+ AlarmAction: alarm.AlarmActionRaise,
+ AlarmTime: (time.Now().UnixNano()),
+ }
+ a.activeAlarms = append(a.activeAlarms, AlarmNotification{thresholdMessage, alarm.AlarmDefinition{}})
+ a.alarmHistory = append(a.alarmHistory, AlarmNotification{thresholdMessage, alarm.AlarmDefinition{}})
+
+ return true
+}
+
+func (a *AlarmManager) UpdateAlarmLists(newAlarm *AlarmNotification) {
/* If maximum number of active alarms is reached, an error log writing is made, and new alarm indicating the problem is raised.
The attempt to raise the alarm next time will be supressed when found as duplicate. */
if (len(a.activeAlarms) >= a.maxActiveAlarms) && (a.exceededActiveAlarmOn == false) {
- app.Logger.Error("active alarm count exceeded maxActiveAlarms threshold")
- actAlarm := a.alarmClient.NewAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, alarm.SeverityWarning, "threshold", "active")
- actAlarmMessage := alarm.AlarmMessage{Alarm: actAlarm, AlarmAction: alarm.AlarmActionRaise, AlarmTime: (time.Now().UnixNano())}
- a.activeAlarms = append(a.activeAlarms, AlarmInformation{actAlarmMessage, alarm.AlarmDefinition{}})
- a.alarmHistory = append(a.alarmHistory, AlarmInformation{actAlarmMessage, alarm.AlarmDefinition{}})
- a.exceededActiveAlarmOn = true
+ app.Logger.Warn("active alarm count exceeded maxActiveAlarms threshold")
+ a.exceededActiveAlarmOn = a.GenerateThresholdAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, "active")
}
if (len(a.alarmHistory) >= a.maxAlarmHistory) && (a.exceededAlarmHistoryOn == false) {
- app.Logger.Error("alarm history count exceeded maxAlarmHistory threshold")
- histAlarm := a.alarmClient.NewAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, alarm.SeverityWarning, "threshold", "history")
- histAlarmMessage := alarm.AlarmMessage{Alarm: histAlarm, AlarmAction: alarm.AlarmActionRaise, AlarmTime: (time.Now().UnixNano())}
- a.activeAlarms = append(a.activeAlarms, AlarmInformation{histAlarmMessage, alarm.AlarmDefinition{}})
- a.alarmHistory = append(a.alarmHistory, AlarmInformation{histAlarmMessage, alarm.AlarmDefinition{}})
- a.exceededAlarmHistoryOn = true
+ app.Logger.Warn("alarm history count exceeded maxAlarmHistory threshold")
+ a.exceededAlarmHistoryOn = a.GenerateThresholdAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, "history")
}
- a.UpdateAlarmFields(newAlarm)
-
// @todo: For now just keep the alarms (both active and history) in-memory. Use SDL later for persistence
a.activeAlarms = append(a.activeAlarms, *newAlarm)
a.alarmHistory = append(a.alarmHistory, *newAlarm)
}
+func (a *AlarmManager) PostAlarm(m *AlarmNotification) (*alert.PostAlertsOK, error) {
+ result, err := json.Marshal(m)
+ if err != nil {
+ app.Logger.Info("json.Marshal failed: %v", err)
+ return nil, err
+ }
+
+ fullUrl := fmt.Sprintf("%s/%s", app.Config.GetString("controls.noma.host"), app.Config.GetString("controls.noma.alarmUrl"))
+ app.Logger.Info("Posting alarm to '%s'", fullUrl)
+
+ resp, err := http.Post(fullUrl, "application/json", bytes.NewReader(result))
+ if err != nil || resp == nil {
+ app.Logger.Info("Unable to post alarm to '%s': %v", fullUrl, err)
+ }
+
+ return nil, err
+}
+
func (a *AlarmManager) GenerateAlertLabels(newAlarm alarm.Alarm, status AlertStatus, alarmTime int64) (models.LabelSet, models.LabelSet) {
alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
amLabels := models.LabelSet{
"status": string(status),
"alertname": alarmDef.AlarmText,
"severity": string(newAlarm.PerceivedSeverity),
- "service": fmt.Sprintf("%s:%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
- "system_name": fmt.Sprintf("RIC:%s:%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
+ "service": fmt.Sprintf("%s/%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
+ "system_name": "RIC",
}
amAnnotations := models.LabelSet{
- "alarm_id": fmt.Sprintf("%d", alarmDef.AlarmId),
- "description": fmt.Sprintf("%d:%s:%s", newAlarm.SpecificProblem, newAlarm.IdentifyingInfo, newAlarm.AdditionalInfo),
- "additional_info": newAlarm.AdditionalInfo,
- "summary": alarmDef.EventType,
- "instructions": alarmDef.OperationInstructions,
- "timestamp": fmt.Sprintf("%s", time.Unix(0, alarmTime).Format("02/01/2006, 15:04:05")),
+ "alarm_id": fmt.Sprintf("%d", alarmDef.AlarmId),
+ "specific_problem": fmt.Sprintf("%d", newAlarm.SpecificProblem),
+ "event_type": alarmDef.EventType,
+ "identifying_info": newAlarm.IdentifyingInfo,
+ "additional_info": newAlarm.AdditionalInfo,
+ "description": fmt.Sprintf("%s:%s", newAlarm.IdentifyingInfo, newAlarm.AdditionalInfo),
+ "instructions": alarmDef.OperationInstructions,
+ "timestamp": fmt.Sprintf("%s", time.Unix(0, alarmTime).Format("02/01/2006, 15:04:05")),
}
return amLabels, amAnnotations
app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.GetAlarmDefinition, "GET")
// Start background timer for re-raising alerts
- a.postClear = sdlcheck
go a.StartAlertTimer()
a.alarmClient, _ = alarm.InitAlarm("SEP", "ALARMMANAGER")
app.RunWithParams(a, sdlcheck)
}
-func NewAlarmManager(amHost string, alertInterval int) *AlarmManager {
+func NewAlarmManager(amHost string, alertInterval int, clearAlarm bool) *AlarmManager {
if alertInterval == 0 {
alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
}
return &AlarmManager{
rmrReady: false,
+ postClear: clearAlarm,
amHost: amHost,
- amBaseUrl: viper.GetString("controls.promAlertManager.baseUrl"),
- amSchemes: []string{viper.GetString("controls.promAlertManager.schemes")},
+ amBaseUrl: app.Config.GetString("controls.promAlertManager.baseUrl"),
+ amSchemes: []string{app.Config.GetString("controls.promAlertManager.schemes")},
alertInterval: alertInterval,
- activeAlarms: make([]AlarmInformation, 0),
- alarmHistory: make([]AlarmInformation, 0),
- uniqueAlarmId: 1,
+ activeAlarms: make([]AlarmNotification, 0),
+ alarmHistory: make([]AlarmNotification, 0),
+ uniqueAlarmId: 0,
maxActiveAlarms: app.Config.GetInt("controls.maxActiveAlarms"),
maxAlarmHistory: app.Config.GetInt("controls.maxAlarmHistory"),
exceededActiveAlarmOn: false,
// Main function
func main() {
- NewAlarmManager("", 0).Run(true)
+ NewAlarmManager("", 0, true).Run(true)
}