2 * Copyright (c) 2020 AT&T Intellectual Property.
3 * Copyright (c) 2020 Nokia.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
17 * This source code is part of the near-RT RIC (RAN Intelligent Controller)
18 * platform project (RICP).
28 clientruntime "github.com/go-openapi/runtime/client"
29 "github.com/go-openapi/strfmt"
30 "github.com/prometheus/alertmanager/api/v2/client"
31 "github.com/prometheus/alertmanager/api/v2/client/alert"
32 "github.com/prometheus/alertmanager/api/v2/models"
33 "github.com/spf13/viper"
35 "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
36 app "gerrit.o-ran-sc.org/r/ric-plt/xapp-frame/pkg/xapp"
39 func (a *AlarmManager) StartAlertTimer() {
40 tick := time.Tick(time.Duration(a.alertInterval) * time.Millisecond)
43 for _, m := range a.activeAlarms {
44 app.Logger.Info("Re-raising alarm: %v", m)
45 a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive))
51 func (a *AlarmManager) Consume(rp *app.RMRParams) (err error) {
52 app.Logger.Info("Message received!")
54 defer app.Rmr.Free(rp.Mbuf)
56 case alarm.RIC_ALARM_UPDATE:
59 app.Logger.Info("Unknown Message Type '%d', discarding", rp.Mtype)
65 func (a *AlarmManager) HandleAlarms(rp *app.RMRParams) (*alert.PostAlertsOK, error) {
66 var m alarm.AlarmMessage
67 app.Logger.Info("Received JSON: %s", rp.Payload)
68 if err := json.Unmarshal(rp.Payload, &m); err != nil {
69 app.Logger.Error("json.Unmarshal failed: %v", err)
72 app.Logger.Info("newAlarm: %v", m)
74 return a.ProcessAlarm(&m)
77 func (a *AlarmManager) ProcessAlarm(m *alarm.AlarmMessage) (*alert.PostAlertsOK, error) {
78 if _, ok := alarm.RICAlarmDefinitions[m.Alarm.SpecificProblem]; !ok {
79 app.Logger.Warn("Alarm (SP='%d') not recognized, suppressing ...", m.Alarm.SpecificProblem)
83 // Suppress duplicate alarms
84 idx, found := a.IsMatchFound(m.Alarm)
85 if found && m.AlarmAction != alarm.AlarmActionClear {
86 app.Logger.Info("Duplicate alarm found, suppressing ...")
90 // Clear alarm if found from active alarm list
91 if m.AlarmAction == alarm.AlarmActionClear {
93 a.activeAlarms = a.RemoveAlarm(a.activeAlarms, idx, "active")
96 return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusResolved))
99 app.Logger.Info("No matching active alarm found, suppressing ...")
103 // New alarm -> update active alarms and post to Alert Manager
104 if m.AlarmAction == alarm.AlarmActionRaise {
105 a.UpdateAlarmLists(m)
106 return a.PostAlert(a.GenerateAlertLabels(m.Alarm, AlertStatusActive))
112 func (a *AlarmManager) IsMatchFound(newAlarm alarm.Alarm) (int, bool) {
113 for i, m := range a.activeAlarms {
114 if m.ManagedObjectId == newAlarm.ManagedObjectId && m.ApplicationId == newAlarm.ApplicationId &&
115 m.SpecificProblem == newAlarm.SpecificProblem && m.IdentifyingInfo == newAlarm.IdentifyingInfo {
122 func (a *AlarmManager) RemoveAlarm(alarms []alarm.AlarmMessage, i int, listName string) []alarm.AlarmMessage {
124 defer a.mutex.Unlock()
126 app.Logger.Info("Alarm '%+v' deleted from the '%s' list", alarms[i], listName)
127 copy(alarms[i:], alarms[i+1:])
128 return alarms[:len(alarms)-1]
131 func (a *AlarmManager) UpdateAlarmLists(newAlarm *alarm.AlarmMessage) {
133 defer a.mutex.Unlock()
135 // If maximum number of active alarms is reached, purge the oldest alarm
136 if len(a.activeAlarms) >= viper.GetInt("controls.maxActiveAlarms") {
137 a.activeAlarms = a.RemoveAlarm(a.activeAlarms, 0, "active")
140 if len(a.alarmHistory) >= viper.GetInt("controls.maxAlarmHistory") {
141 a.alarmHistory = a.RemoveAlarm(a.alarmHistory, 0, "history")
144 // @todo: For now just keep the alarms (both active and history) in-memory. Use SDL later for persistence
145 a.activeAlarms = append(a.activeAlarms, *newAlarm)
146 a.alarmHistory = append(a.alarmHistory, *newAlarm)
149 func (a *AlarmManager) GenerateAlertLabels(newAlarm alarm.Alarm, status AlertStatus) (models.LabelSet, models.LabelSet) {
150 alarmDef := alarm.RICAlarmDefinitions[newAlarm.SpecificProblem]
151 amLabels := models.LabelSet{
152 "status": string(status),
153 "alertname": alarmDef.AlarmText,
154 "severity": string(newAlarm.PerceivedSeverity),
155 "service": fmt.Sprintf("%s:%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
156 "system_name": fmt.Sprintf("RIC:%s:%s", newAlarm.ManagedObjectId, newAlarm.ApplicationId),
158 amAnnotations := models.LabelSet{
159 "alarm_id": fmt.Sprintf("%d", alarmDef.AlarmId),
160 "description": fmt.Sprintf("%d:%s:%s", newAlarm.SpecificProblem, newAlarm.IdentifyingInfo, newAlarm.AdditionalInfo),
161 "additional_info": newAlarm.AdditionalInfo,
162 "summary": alarmDef.EventType,
163 "instructions": alarmDef.OperationInstructions,
166 return amLabels, amAnnotations
169 func (a *AlarmManager) NewAlertmanagerClient() *client.Alertmanager {
170 cr := clientruntime.New(a.amHost, a.amBaseUrl, a.amSchemes)
171 return client.New(cr, strfmt.Default)
174 func (a *AlarmManager) PostAlert(amLabels, amAnnotations models.LabelSet) (*alert.PostAlertsOK, error) {
175 pa := &models.PostableAlert{
177 GeneratorURL: strfmt.URI(""),
180 Annotations: amAnnotations,
182 alertParams := alert.NewPostAlertsParams().WithAlerts(models.PostableAlerts{pa})
184 app.Logger.Info("Posting alerts: labels: %+v, annotations: %+v", amLabels, amAnnotations)
185 ok, err := a.NewAlertmanagerClient().Alert.PostAlerts(alertParams)
187 app.Logger.Error("Posting alerts to '%s/%s' failed with error: %v", a.amHost, a.amBaseUrl, err)
192 func (a *AlarmManager) StatusCB() bool {
194 app.Logger.Info("RMR not ready yet!")
200 func (a *AlarmManager) Run(sdlcheck bool) {
201 app.Logger.SetMdc("alarmManager", fmt.Sprintf("%s:%s", Version, Hash))
202 app.SetReadyCB(func(d interface{}) { a.rmrReady = true }, true)
203 app.Resource.InjectStatusCb(a.StatusCB)
205 app.Resource.InjectRoute("/ric/v1/alarms", a.RaiseAlarm, "POST")
206 app.Resource.InjectRoute("/ric/v1/alarms", a.ClearAlarm, "DELETE")
207 app.Resource.InjectRoute("/ric/v1/alarms/active", a.GetActiveAlarms, "GET")
208 app.Resource.InjectRoute("/ric/v1/alarms/history", a.GetAlarmHistory, "GET")
210 // Start background timer for re-raising alerts
211 a.postClear = sdlcheck
212 go a.StartAlertTimer()
214 app.RunWithParams(a, sdlcheck)
217 func NewAlarmManager(amHost string, alertInterval int) *AlarmManager {
218 if alertInterval == 0 {
219 alertInterval = viper.GetInt("controls.promAlertManager.alertInterval")
223 amHost = viper.GetString("controls.promAlertManager.address")
226 return &AlarmManager{
229 amBaseUrl: viper.GetString("controls.promAlertManager.baseUrl"),
230 amSchemes: []string{viper.GetString("controls.promAlertManager.schemes")},
231 alertInterval: alertInterval,
232 activeAlarms: make([]alarm.AlarmMessage, 0),
233 alarmHistory: make([]alarm.AlarmMessage, 0),
239 NewAlarmManager("", 0).Run(true)