2 * Copyright (c) 2020 AT&T Intellectual Property.
3 * Copyright (c) 2020 Nokia.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 * This source code is part of the near-RT RIC (RAN Intelligent Controller)
18 * platform project (RICP).
28 clientruntime "github.com/go-openapi/runtime/client"
29 "github.com/go-openapi/strfmt"
30 "github.com/prometheus/alertmanager/api/v2/client"
31 "github.com/prometheus/alertmanager/api/v2/client/alert"
32 "github.com/prometheus/alertmanager/api/v2/models"
33 "github.com/spf13/viper"
35 "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
36 app "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
39 func (a *AlarmManager) StartAlertTimer() {
40 tick := time.Tick(time.Duration(a.alertInterval) * time.Millisecond)
43 for _, m := range a.activeAlarms {
44 app.Logger.Info("Re-raising alarm: %v", m)
45 a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime))
51 func (a *AlarmManager) Consume(rp *app.RMRParams) (err error) {
52 app.Logger.Info("Message received!")
54 defer app.Rmr.Free(rp.Mbuf)
56 case alarm.RIC_ALARM_UPDATE:
59 app.Logger.Info("Unknown Message Type '%d', discarding", rp.Mtype)
65 func (a *AlarmManager) HandleAlarms(rp *app.RMRParams) (*alert.PostAlertsOK, error) {
66 var m alarm.AlarmMessage
67 app.Logger.Info("Received JSON: %s", rp.Payload)
68 if err := json.Unmarshal(rp.Payload, &m); err != nil {
69 app.Logger.Error("json.Unmarshal failed: %v", err)
72 app.Logger.Info("newAlarm: %v", m)
74 return a.ProcessAlarm(&m)
77 func (a *AlarmManager) ProcessAlarm(m *alarm.AlarmMessage) (*alert.PostAlertsOK, error) {
78 if _, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok {
79 app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem)
83 // Suppress duplicate alarms
84 idx, found := a.IsMatchFound(m.Alarm)
85 if found && m.AlarmAction == alarm.AlarmActionRaise {
86 app.Logger.Info("Duplicate alarm found, suppressing ...")
87 if m.PerceivedSeverity == a.activeAlarms[idx].PerceivedSeverity {
88 // Duplicate with same severity found
91 // Remove duplicate with different severity
92 a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
97 // Clear alarm if found from active alarm list
98 if m.AlarmAction == alarm.AlarmActionClear {
100 a.alarmHistory = append(a.alarmHistory, *m)
101 a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
104 return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusResolved, m.AlarmTime))
107 app.Logger.Info("No matching active alarm found, suppressing ...")
111 // New alarm -> update active alarms and post to Alert Manager
112 if m.AlarmAction == alarm.AlarmActionRaise {
113 a.UpdateAlarmLists(m)
114 return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive, m.AlarmTime))
120 func (a *AlarmManager) IsMatchFound(newAlarm alarm.Alarm) (int, bool) {
121 for i, m := range a.activeAlarms {
122 if m.ManagedObjectId == newAlarm.ManagedObjectId && m.ApplicationId == newAlarm.ApplicationId &&
123 m.SpecificProblem == newAlarm.SpecificProblem && m.IdentifyingInfo == newAlarm.IdentifyingInfo {
130 func (a *AlarmManager) RemoveAlarm(alarms []alarm.AlarmMessage, i int, listName string) []alarm.AlarmMessage {
132 defer a.mutex.Unlock()
134 app.Logger.Info("Alarm '%+v' deleted from the '%s' list", alarms[i], listName)
135 copy(alarms[i:], alarms[i+1:])
136 return alarms[:len(alarms)-1]
139 func (a *AlarmManager) UpdateAlarmLists(newAlarm *alarm.AlarmMessage) {
141 defer a.mutex.Unlock()
143 /* If maximum number of active alarms is reached, an error log writing is made, and new alarm indicating the problem is raised.
144 The attempt to raise the alarm next time will be supressed when found as duplicate. */
145 if len(a.activeAlarms) >= a.maxActiveAlarms {
146 app.Logger.Error("active alarm count exceeded maxActiveAlarms threshold")
147 actAlarm := a.alarmClient.NewAlarm(alarm.ACTIVE_ALARM_EXCEED_MAX_THRESHOLD, alarm.SeverityWarning, "clear alarms or raise threshold", "active alarms full")
148 actAlarmMessage := alarm.AlarmMessage{Alarm: actAlarm, AlarmAction: alarm.AlarmActionRaise, AlarmTime: (time.Now().UnixNano())}
149 a.activeAlarms = append(a.activeAlarms, actAlarmMessage)
150 a.alarmHistory = append(a.alarmHistory, actAlarmMessage)
153 if len(a.alarmHistory) >= a.maxAlarmHistory {
154 app.Logger.Error("alarm history count exceeded maxAlarmHistory threshold")
155 histAlarm := a.alarmClient.NewAlarm(alarm.ALARM_HISTORY_EXCEED_MAX_THRESHOLD, alarm.SeverityWarning, "clear alarms or raise threshold", "alarm history full")
156 histAlarmMessage := alarm.AlarmMessage{Alarm: histAlarm, AlarmAction: alarm.AlarmActionRaise, AlarmTime: (time.Now().UnixNano())}
157 a.activeAlarms = append(a.activeAlarms, histAlarmMessage)
158 a.alarmHistory = append(a.alarmHistory, histAlarmMessage)
161 // @todo: For now just keep the alarms (both active and history) in-memory. Use SDL later for persistence
162 a.activeAlarms = append(a.activeAlarms, *newAlarm)
163 a.alarmHistory = append(a.alarmHistory, *newAlarm)
166 func (a *AlarmManager) GenerateAlertLabels(newAlarm alarm.Alarm, status AlertStatus, alarmTime int64) (models.LabelSet, models.LabelSet) {
167 alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
168 amLabels := models.LabelSet{
169 "status": string(status),
170 "alertname": alarmDef.AlarmText,
171 "severity": string(newAlarm.PerceivedSeverity),
172 "service": fmt.Sprintf("%s:%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
173 "system_name": fmt.Sprintf("RIC:%s:%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
175 amAnnotations := models.LabelSet{
176 "alarm_id": fmt.Sprintf("%d", alarmDef.AlarmId),
177 "description": fmt.Sprintf("%d:%s:%s", newAlarm.SpecificProblem, newAlarm.IdentifyingInfo, newAlarm.AdditionalInfo),
178 "additional_info": newAlarm.AdditionalInfo,
179 "summary": alarmDef.EventType,
180 "instructions": alarmDef.OperationInstructions,
181 "timestamp": fmt.Sprintf("%s", time.Unix(0, alarmTime).Format("02/01/2006, 15:04:05")),
184 return amLabels, amAnnotations
187 func (a *AlarmManager) NewAlertmanagerClient() *client.Alertmanager {
188 cr := clientruntime.New(a.amHost, a.amBaseUrl, a.amSchemes)
189 return client.New(cr, strfmt.Default)
192 func (a *AlarmManager) PostAlert(amLabels, amAnnotations models.LabelSet) (*alert.PostAlertsOK, error) {
193 pa := &models.PostableAlert{
195 GeneratorURL: strfmt.URI(""),
198 Annotations: amAnnotations,
200 alertParams := alert.NewPostAlertsParams().WithAlerts(models.PostableAlerts{pa})
202 app.Logger.Info("Posting alerts: labels: %+v, annotations: %+v", amLabels, amAnnotations)
203 ok, err := a.NewAlertmanagerClient().Alert.PostAlerts(alertParams)
205 app.Logger.Error("Posting alerts to '%s/%s' failed with error: %v", a.amHost, a.amBaseUrl, err)
210 func (a *AlarmManager) StatusCB() bool {
212 app.Logger.Info("RMR not ready yet!")
218 func (a *AlarmManager) ConfigChangeCB(configparam string) {
220 a.maxActiveAlarms = app.Config.GetInt("controls.maxActiveAlarms")
221 a.maxAlarmHistory = app.Config.GetInt("controls.maxAlarmHistory")
223 app.Logger.Debug("ConfigChangeCB: maxActiveAlarms %v", a.maxActiveAlarms)
224 app.Logger.Debug("ConfigChangeCB: maxAlarmHistory = %v", a.maxAlarmHistory)
229 func (a *AlarmManager) Run(sdlcheck bool) {
230 app.Logger.SetMdc("alarmManager", fmt.Sprintf("%s:%s", Version, Hash))
231 app.SetReadyCB(func(d interface{}) { a.rmrReady = true }, true)
232 app.Resource.InjectStatusCb(a.StatusCB)
233 app.AddConfigChangeListener(a.ConfigChangeCB)
235 alarm.RICAlarmDefinitions = make(map[int]*alarm.AlarmDefinition)
237 app.Resource.InjectRoute("/ric/v1/alarms", a.RaiseAlarm, "POST")
238 app.Resource.InjectRoute("/ric/v1/alarms", a.ClearAlarm, "DELETE")
239 app.Resource.InjectRoute("/ric/v1/alarms/active", a.GetActiveAlarms, "GET")
240 app.Resource.InjectRoute("/ric/v1/alarms/history", a.GetAlarmHistory, "GET")
241 app.Resource.InjectRoute("/ric/v1/alarms/config", a.SetAlarmConfig, "POST")
242 app.Resource.InjectRoute("/ric/v1/alarms/config", a.GetAlarmConfig, "GET")
243 app.Resource.InjectRoute("/ric/v1/alarms/define", a.SetAlarmDefinition, "POST")
244 app.Resource.InjectRoute("/ric/v1/alarms/define/{alarmId}", a.DeleteAlarmDefinition, "DELETE")
245 app.Resource.InjectRoute("/ric/v1/alarms/define", a.GetAlarmDefinition, "GET")
247 // Start background timer for re-raising alerts
248 a.postClear = sdlcheck
249 go a.StartAlertTimer()
250 a.alarmClient, _ = alarm.InitAlarm("SEP", "ALARMMANAGER")
252 app.RunWithParams(a, sdlcheck)
255 func NewAlarmManager(amHost string, alertInterval int) *AlarmManager {
256 if alertInterval == 0 {
257 alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
261 amHost = viper.GetString("controls.promAlertManager.address")
264 return &AlarmManager{
267 amBaseUrl: viper.GetString("controls.promAlertManager.baseUrl"),
268 amSchemes: []string{viper.GetString("controls.promAlertManager.schemes")},
269 alertInterval: alertInterval,
270 activeAlarms: make([]alarm.AlarmMessage, 0),
271 alarmHistory: make([]alarm.AlarmMessage, 0),
272 maxActiveAlarms: app.Config.GetInt("controls.maxActiveAlarms"),
273 maxAlarmHistory: app.Config.GetInt("controls.maxAlarmHistory"),
279 NewAlarmManager("", 0).Run(true)