2 * Copyright (c) 2020 AT&T Intellectual Property.
3 * Copyright (c) 2020 Nokia.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 * This source code is part of the near-RT RIC (RAN Intelligent Controller)
18 * platform project (RICP).
28 clientruntime "github.com/go-openapi/runtime/client"
29 "github.com/go-openapi/strfmt"
30 "github.com/prometheus/alertmanager/api/v2/client"
31 "github.com/prometheus/alertmanager/api/v2/client/alert"
32 "github.com/prometheus/alertmanager/api/v2/models"
33 "github.com/spf13/viper"
35 "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
36 app "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
39 func (a *AlarmManager) StartAlertTimer() {
40 tick := time.Tick(time.Duration(a.alertInterval) * time.Millisecond)
43 for _, m := range a.activeAlarms {
44 app.Logger.Info("Re-raising alarm: %v", m)
45 a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive))
51 func (a *AlarmManager) Consume(rp *app.RMRParams) (err error) {
52 app.Logger.Info("Message received!")
54 defer app.Rmr.Free(rp.Mbuf)
56 case alarm.RIC_ALARM_UPDATE:
59 app.Logger.Info("Unknown Message Type '%d', discarding", rp.Mtype)
65 func (a *AlarmManager) HandleAlarms(rp *app.RMRParams) (*alert.PostAlertsOK, error) {
66 var m alarm.AlarmMessage
67 app.Logger.Info("Received JSON: %s", rp.Payload)
68 if err := json.Unmarshal(rp.Payload, &m); err != nil {
69 app.Logger.Error("json.Unmarshal failed: %v", err)
72 app.Logger.Info("newAlarm: %v", m)
74 return a.ProcessAlarm(&m)
77 func (a *AlarmManager) ProcessAlarm(m *alarm.AlarmMessage) (*alert.PostAlertsOK, error) {
78 if _, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok {
79 app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem)
83 // Suppress duplicate alarms
84 idx, found := a.IsMatchFound(m.Alarm)
85 if found && m.AlarmAction != alarm.AlarmActionClear {
86 app.Logger.Info("Duplicate alarm found, suppressing ...")
90 // Clear alarm if found from active alarm list
91 if m.AlarmAction == alarm.AlarmActionClear {
93 a.alarmHistory = append(a.alarmHistory, *m)
94 a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
97 return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusResolved))
100 app.Logger.Info("No matching active alarm found, suppressing ...")
104 // New alarm -> update active alarms and post to Alert Manager
105 if m.AlarmAction == alarm.AlarmActionRaise {
106 a.UpdateAlarmLists(m)
107 return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive))
113 func (a *AlarmManager) IsMatchFound(newAlarm alarm.Alarm) (int, bool) {
114 for i, m := range a.activeAlarms {
115 if m.ManagedObjectId == newAlarm.ManagedObjectId && m.ApplicationId == newAlarm.ApplicationId &&
116 m.SpecificProblem == newAlarm.SpecificProblem && m.IdentifyingInfo == newAlarm.IdentifyingInfo {
123 func (a *AlarmManager) RemoveAlarm(alarms []alarm.AlarmMessage, i int, listName string) []alarm.AlarmMessage {
125 defer a.mutex.Unlock()
127 app.Logger.Info("Alarm '%+v' deleted from the '%s' list", alarms[i], listName)
128 copy(alarms[i:], alarms[i+1:])
129 return alarms[:len(alarms)-1]
132 func (a *AlarmManager) UpdateAlarmLists(newAlarm *alarm.AlarmMessage) {
134 defer a.mutex.Unlock()
136 // If maximum number of active alarms is reached, purge the oldest alarm
137 if len(a.activeAlarms) >= viper.GetInt("controls.maxActiveAlarms") {
138 a.activeAlarms = a.RemoveAlarm(a.activeAlarms, 0, "active")
141 if len(a.alarmHistory) >= viper.GetInt("controls.maxAlarmHistory") {
142 a.alarmHistory = a.RemoveAlarm(a.alarmHistory, 0, "history")
145 // @todo: For now just keep the alarms (both active and history) in-memory. Use SDL later for persistence
146 a.activeAlarms = append(a.activeAlarms, *newAlarm)
147 a.alarmHistory = append(a.alarmHistory, *newAlarm)
150 func (a *AlarmManager) GenerateAlertLabels(newAlarm alarm.Alarm, status AlertStatus) (models.LabelSet, models.LabelSet) {
151 alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
152 amLabels := models.LabelSet{
153 "status": string(status),
154 "alertname": alarmDef.AlarmText,
155 "severity": string(newAlarm.PerceivedSeverity),
156 "service": fmt.Sprintf("%s:%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
157 "system_name": fmt.Sprintf("RIC:%s:%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
159 amAnnotations := models.LabelSet{
160 "alarm_id": fmt.Sprintf("%d", alarmDef.AlarmId),
161 "description": fmt.Sprintf("%d:%s:%s", newAlarm.SpecificProblem, newAlarm.IdentifyingInfo, newAlarm.AdditionalInfo),
162 "additional_info": newAlarm.AdditionalInfo,
163 "summary": alarmDef.EventType,
164 "instructions": alarmDef.OperationInstructions,
167 return amLabels, amAnnotations
170 func (a *AlarmManager) NewAlertmanagerClient() *client.Alertmanager {
171 cr := clientruntime.New(a.amHost, a.amBaseUrl, a.amSchemes)
172 return client.New(cr, strfmt.Default)
175 func (a *AlarmManager) PostAlert(amLabels, amAnnotations models.LabelSet) (*alert.PostAlertsOK, error) {
176 pa := &models.PostableAlert{
178 GeneratorURL: strfmt.URI(""),
181 Annotations: amAnnotations,
183 alertParams := alert.NewPostAlertsParams().WithAlerts(models.PostableAlerts{pa})
185 app.Logger.Info("Posting alerts: labels: %+v, annotations: %+v", amLabels, amAnnotations)
186 ok, err := a.NewAlertmanagerClient().Alert.PostAlerts(alertParams)
188 app.Logger.Error("Posting alerts to '%s/%s' failed with error: %v", a.amHost, a.amBaseUrl, err)
193 func (a *AlarmManager) StatusCB() bool {
195 app.Logger.Info("RMR not ready yet!")
201 func (a *AlarmManager) Run(sdlcheck bool) {
202 app.Logger.SetMdc("alarmManager", fmt.Sprintf("%s:%s", Version, Hash))
203 app.SetReadyCB(func(d interface{}) { a.rmrReady = true }, true)
204 app.Resource.InjectStatusCb(a.StatusCB)
206 app.Resource.InjectRoute("/ric/v1/alarms", a.RaiseAlarm, "POST")
207 app.Resource.InjectRoute("/ric/v1/alarms", a.ClearAlarm, "DELETE")
208 app.Resource.InjectRoute("/ric/v1/alarms/active", a.GetActiveAlarms, "GET")
209 app.Resource.InjectRoute("/ric/v1/alarms/history", a.GetAlarmHistory, "GET")
211 // Start background timer for re-raising alerts
212 a.postClear = sdlcheck
213 go a.StartAlertTimer()
215 app.RunWithParams(a, sdlcheck)
218 func NewAlarmManager(amHost string, alertInterval int) *AlarmManager {
219 if alertInterval == 0 {
220 alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
224 amHost = viper.GetString("controls.promAlertManager.address")
227 return &AlarmManager{
230 amBaseUrl: viper.GetString("controls.promAlertManager.baseUrl"),
231 amSchemes: []string{viper.GetString("controls.promAlertManager.schemes")},
232 alertInterval: alertInterval,
233 activeAlarms: make([]alarm.AlarmMessage, 0),
234 alarmHistory: make([]alarm.AlarmMessage, 0),
240 NewAlarmManager("", 0).Run(true)