2 * Copyright (c) 2020 AT&T Intellectual Property.
3 * Copyright (c) 2020 Nokia.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 * This source code is part of the near-RT RIC (RAN Intelligent Controller)
18 * platform project (RICP).
28 clientruntime "github.com/go-openapi/runtime/client"
29 "github.com/go-openapi/strfmt"
30 "github.com/prometheus/alertmanager/api/v2/client"
31 "github.com/prometheus/alertmanager/api/v2/client/alert"
32 "github.com/prometheus/alertmanager/api/v2/models"
33 "github.com/spf13/viper"
35 "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
36 app "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
39 func (a *AlarmManager) StartAlertTimer() {
40 tick := time.Tick(time.Duration(a.alertInterval) * time.Millisecond)
43 for _, m := range a.activeAlarms {
44 app.Logger.Info("Re-raising alarm: %v", m)
45 a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime))
51 func (a *AlarmManager) Consume(rp *app.RMRParams) (err error) {
52 app.Logger.Info("Message received!")
54 defer app.Rmr.Free(rp.Mbuf)
56 case alarm.RIC_ALARM_UPDATE:
59 app.Logger.Info("Unknown Message Type '%d', discarding", rp.Mtype)
65 func (a *AlarmManager) HandleAlarms(rp *app.RMRParams) (*alert.PostAlertsOK, error) {
66 var m alarm.AlarmMessage
67 app.Logger.Info("Received JSON: %s", rp.Payload)
68 if err := json.Unmarshal(rp.Payload, &m); err != nil {
69 app.Logger.Error("json.Unmarshal failed: %v", err)
72 app.Logger.Info("newAlarm: %v", m)
74 return a.ProcessAlarm(&m)
77 func (a *AlarmManager) ProcessAlarm(m *alarm.AlarmMessage) (*alert.PostAlertsOK, error) {
78 if _, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok {
79 app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem)
83 // Suppress duplicate alarms
84 idx, found := a.IsMatchFound(m.Alarm)
85 if found && m.AlarmAction != alarm.AlarmActionClear {
86 app.Logger.Info("Duplicate alarm found, suppressing ...")
90 // Clear alarm if found from active alarm list
91 if m.AlarmAction == alarm.AlarmActionClear {
93 a.alarmHistory = append(a.alarmHistory, *m)
94 a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
97 return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusResolved, m.AlarmTime))
100 app.Logger.Info("No matching active alarm found, suppressing ...")
104 // New alarm -> update active alarms and post to Alert Manager
105 if m.AlarmAction == alarm.AlarmActionRaise {
106 a.UpdateAlarmLists(m)
107 return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime))
113 func (a *AlarmManager) IsMatchFound(newAlarm alarm.Alarm) (int, bool) {
114 for i, m := range a.activeAlarms {
115 if m.ManagedObjectId == newAlarm.ManagedObjectId && m.ApplicationId == newAlarm.ApplicationId &&
116 m.SpecificProblem == newAlarm.SpecificProblem && m.IdentifyingInfo == newAlarm.IdentifyingInfo {
123 func (a *AlarmManager) RemoveAlarm(alarms []alarm.AlarmMessage, i int, listName string) []alarm.AlarmMessage {
125 defer a.mutex.Unlock()
127 app.Logger.Info("Alarm '%+v' deleted from the '%s' list", alarms[i], listName)
128 copy(alarms[i:], alarms[i+1:])
129 return alarms[:len(alarms)-1]
132 func (a *AlarmManager) UpdateAlarmLists(newAlarm *alarm.AlarmMessage) {
134 defer a.mutex.Unlock()
136 /* If maximum number of active alarms is reached, an error log writing is made, and new alarm indicating the problem is raised.
137 The attempt to raise the alarm next time will be supressed when found as duplicate. */
138 if len(a.activeAlarms) >= a.maxActiveAlarms {
139 app.Logger.Error("active alarm count exceeded maxActiveAlarms threshold")
140 actAlarm := a.alarmClient.NewAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, alarm.SeverityWarning, "clear alarms or raise threshold", "active alarms full")
141 actAlarmMessage := alarm.AlarmMessage{Alarm: actAlarm, AlarmAction: alarm.AlarmActionRaise, AlarmTime: (time.Now().UnixNano())}
142 a.activeAlarms = append(a.activeAlarms, actAlarmMessage)
143 a.alarmHistory = append(a.alarmHistory, actAlarmMessage)
146 if len(a.alarmHistory) >= a.maxAlarmHistory {
147 app.Logger.Error("alarm history count exceeded maxAlarmHistory threshold")
148 histAlarm := a.alarmClient.NewAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, alarm.SeverityWarning, "clear alarms or raise threshold", "alarm history full")
149 histAlarmMessage := alarm.AlarmMessage{Alarm: histAlarm, AlarmAction: alarm.AlarmActionRaise, AlarmTime: (time.Now().UnixNano())}
150 a.activeAlarms = append(a.activeAlarms, histAlarmMessage)
151 a.alarmHistory = append(a.alarmHistory, histAlarmMessage)
154 // @todo: For now just keep the alarms (both active and history) in-memory. Use SDL later for persistence
155 a.activeAlarms = append(a.activeAlarms, *newAlarm)
156 a.alarmHistory = append(a.alarmHistory, *newAlarm)
159 func (a *AlarmManager) GenerateAlertLabels(newAlarm alarm.Alarm, status AlertStatus, alarmTime int64) (models.LabelSet, models.LabelSet) {
160 alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
161 amLabels := models.LabelSet{
162 "status": string(status),
163 "alertname": alarmDef.AlarmText,
164 "severity": string(newAlarm.PerceivedSeverity),
165 "service": fmt.Sprintf("%s:%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
166 "system_name": fmt.Sprintf("RIC:%s:%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
168 amAnnotations := models.LabelSet{
169 "alarm_id": fmt.Sprintf("%d", alarmDef.AlarmId),
170 "description": fmt.Sprintf("%d:%s:%s", newAlarm.SpecificProblem, newAlarm.IdentifyingInfo, newAlarm.AdditionalInfo),
171 "additional_info": newAlarm.AdditionalInfo,
172 "summary": alarmDef.EventType,
173 "instructions": alarmDef.OperationInstructions,
174 "timestamp": fmt.Sprintf("%s", time.Unix(0, alarmTime).Format("02/01/2006, 15:04:05")),
177 return amLabels, amAnnotations
180 func (a *AlarmManager) NewAlertmanagerClient() *client.Alertmanager {
181 cr := clientruntime.New(a.amHost, a.amBaseUrl, a.amSchemes)
182 return client.New(cr, strfmt.Default)
185 func (a *AlarmManager) PostAlert(amLabels, amAnnotations models.LabelSet) (*alert.PostAlertsOK, error) {
186 pa := &models.PostableAlert{
188 GeneratorURL: strfmt.URI(""),
191 Annotations: amAnnotations,
193 alertParams := alert.NewPostAlertsParams().WithAlerts(models.PostableAlerts{pa})
195 app.Logger.Info("Posting alerts: labels: %+v, annotations: %+v", amLabels, amAnnotations)
196 ok, err := a.NewAlertmanagerClient().Alert.PostAlerts(alertParams)
198 app.Logger.Error("Posting alerts to '%s/%s' failed with error: %v", a.amHost, a.amBaseUrl, err)
203 func (a *AlarmManager) StatusCB() bool {
205 app.Logger.Info("RMR not ready yet!")
211 func (a *AlarmManager) ConfigChangeCB(configparam string) {
213 a.maxActiveAlarms = app.Config.GetInt("controls.maxActiveAlarms")
214 a.maxAlarmHistory = app.Config.GetInt("controls.maxAlarmHistory")
216 app.Logger.Debug("ConfigChangeCB: maxActiveAlarms %v", a.maxActiveAlarms)
217 app.Logger.Debug("ConfigChangeCB: maxAlarmHistory = %v", a.maxAlarmHistory)
222 func (a *AlarmManager) Run(sdlcheck bool) {
223 app.Logger.SetMdc("alarmManager", fmt.Sprintf("%s:%s", Version, Hash))
224 app.SetReadyCB(func(d interface{}) { a.rmrReady = true }, true)
225 app.Resource.InjectStatusCb(a.StatusCB)
226 app.AddConfigChangeListener(a.ConfigChangeCB)
228 alarm.RICAlarmDefinitions = make(map[int]*alarm.AlarmDefinition)
230 app.Resource.InjectRoute("/ric/v1/alarms", a.RaiseAlarm, "POST")
231 app.Resource.InjectRoute("/ric/v1/alarms", a.ClearAlarm, "DELETE")
232 app.Resource.InjectRoute("/ric/v1/alarms/active", a.GetActiveAlarms, "GET")
233 app.Resource.InjectRoute("/ric/v1/alarms/history", a.GetAlarmHistory, "GET")
234 app.Resource.InjectRoute("/ric/v1/alarms/config", a.SetAlarmConfig, "POST")
235 app.Resource.InjectRoute("/ric/v1/alarms/config", a.GetAlarmConfig, "GET")
236 app.Resource.InjectRoute("/ric/v1/alarms/define", a.SetAlarmDefinition, "POST")
237 app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.DeleteAlarmDefinition, "DELETE")
238 app.Resource.InjectRoute("/ric/v1/alarms/define", a.GetAlarmDefinition, "GET")
240 // Start background timer for re-raising alerts
241 a.postClear = sdlcheck
242 go a.StartAlertTimer()
243 a.alarmClient, _ = alarm.InitAlarm("SEP", "ALARMMANAGER")
245 app.RunWithParams(a, sdlcheck)
248 func NewAlarmManager(amHost string, alertInterval int) *AlarmManager {
249 if alertInterval == 0 {
250 alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
254 amHost = viper.GetString("controls.promAlertManager.address")
257 return &AlarmManager{
260 amBaseUrl: viper.GetString("controls.promAlertManager.baseUrl"),
261 amSchemes: []string{viper.GetString("controls.promAlertManager.schemes")},
262 alertInterval: alertInterval,
263 activeAlarms: make([]alarm.AlarmMessage, 0),
264 alarmHistory: make([]alarm.AlarmMessage, 0),
265 maxActiveAlarms: app.Config.GetInt("controls.maxActiveAlarms"),
266 maxAlarmHistory: app.Config.GetInt("controls.maxAlarmHistory"),
272 NewAlarmManager("", 0).Run(true)