LN0739_FM_RF13: support interface to external Prometheus alert manager 56/4856/1
authorAnssi Mannila <anssi.mannila@nokia.com>
Mon, 19 Oct 2020 08:36:26 +0000 (11:36 +0300)
committerAnssi Mannila <anssi.mannila@nokia.com>
Mon, 19 Oct 2020 08:36:39 +0000 (11:36 +0300)
Change-Id: Ia255950c07f3abc51edbe139f4856cfd6c03448f
Signed-off-by: Anssi Mannila <anssi.mannila@nokia.com>
build/Dockerfile
build/build_ubuntu.sh
build/run_local.sh
cli/alarm-cli.go
docs/user-guide.rst
manager/cmd/manager_test.go
testresources/perf-alarm-definition.json [moved from perfresources/perf-alarm-definition.json with 100% similarity]
testresources/perf-alarm-object.json [moved from perfresources/perf-alarm-object.json with 100% similarity]
testresources/prometheus-alerts.json [new file with mode: 0644]

index 7e9ebc8..b2d0884 100755 (executable)
@@ -36,8 +36,8 @@ FROM ubuntu:18.04
 COPY --from=ubuntu-alarm-manager /go/src/am/build/run.sh /
 COPY --from=ubuntu-alarm-manager /go/src/am/manager/alarm-manager /
 COPY --from=ubuntu-alarm-manager /go/src/am/cli/alarm-cli /
-COPY --from=ubuntu-alarm-manager /go/src/am/perfresources/perf-alarm-object.json /
-COPY --from=ubuntu-alarm-manager /go/src/am/perfresources/perf-alarm-definition.json /
+COPY --from=ubuntu-alarm-manager /go/src/am/testresources/perf-alarm-object.json /
+COPY --from=ubuntu-alarm-manager /go/src/am/testresources/perf-alarm-definition.json /
 COPY --from=ubuntu-alarm-manager /go/src/am/config/* /
 COPY --from=ubuntu-alarm-manager /usr/local/lib /usr/local/lib
 COPY --from=ubuntu-alarm-manager /go/src/am/definitions/* / 
index 79e5a8c..26bc0e3 100755 (executable)
@@ -44,8 +44,8 @@ export RMR_SEED_RT=../config/uta_rtg.rt
 
 # xApp stuff
 export DEF_FILE=../definitions/alarm-definition.json
-export PERF_DEF_FILE=../perfresources/perf-alarm-definition.json
-export PERF_OBJ_FILE=../perfresources/perf-alarm-object.json
+export PERF_DEF_FILE=../testresources/perf-alarm-definition.json
+export PERF_OBJ_FILE=../testresources/perf-alarm-object.json
 
 GO111MODULE=on GO_ENABLED=0 GOOS=linux
 
@@ -61,6 +61,10 @@ hash=$(git rev-parse --short HEAD || true)
 
 ROOT_DIR=$PWD
 
+
+# compile the CLI
+cd ${ROOT_DIR}/cli && go build -a -installsuffix cgo alarm-cli.go
+
 # Build
 cd ${ROOT_DIR}/manager && go build -a -installsuffix cgo -ldflags "-X main.Version=$tag -X main.Hash=$hash" -o alarm-manager ./cmd/*.go
 
@@ -70,7 +74,4 @@ cd ${ROOT_DIR}/alarm && go test . -v -coverprofile cover.out
 # And for the Alarm Manager
 cd ${ROOT_DIR}/manager && go test -v -p 1 -coverprofile cover.out ./cmd/ -c -o ./manager_test && ./manager_test
 
-# Finally compile the CLI
-cd ${ROOT_DIR}/cli && go build -a -installsuffix cgo alarm-cli.go
-
 echo "--> build_ubuntu.sh ends"
index ddd4524..24fdd1a 100755 (executable)
@@ -25,7 +25,7 @@
 export RMR_SEED_RT=$PWD/config/uta_rtg.rt
 export RMR_SRC_ID="service-ricplt-alarmmanager-rmr.ricplt"
 export DEF_FILE=$PWD/definitions/alarm-definition.json
-export PERF_DEF_FILE=$PWD/perfresources/perf-alarm-definition.json
-export PERF_OBJ_FILE=$PWD/perfresources/perf-alarm-object.json
+export PERF_DEF_FILE=$PWD/testresources/perf-alarm-definition.json
+export PERF_OBJ_FILE=$PWD/testresources/perf-alarm-object.json
 
 $PWD/manager/alarm-manager -f $PWD/config/config-file.json
index afe9da9..f666292 100755 (executable)
@@ -8,12 +8,17 @@ import (
        "net/http"
        "os"
        "strconv"
+       "sync"
        "time"
 
        "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm"
+       clientruntime "github.com/go-openapi/runtime/client"
+       "github.com/go-openapi/strfmt"
        "github.com/jedib0t/go-pretty/table"
+       "github.com/prometheus/alertmanager/api/v2/client"
+       "github.com/prometheus/alertmanager/api/v2/client/alert"
        "github.com/thatisuday/commando"
-       "sync"
+       "github.com/spf13/viper"
 )
 
 type CliAlarmDefinitions struct {
@@ -153,6 +158,20 @@ func main() {
                        conductperformancetest(flags)
                })
 
+       // Get alerts from Prometheus Alert Manager
+       commando.
+               Register("gapam").
+               SetShortDescription("Get alerts from Prometheus Alert Manager").
+               AddFlag("active", "Active alerts in Prometheus Alert Manager", commando.Bool, true).
+               AddFlag("inhibited", "Inhibited alerts in Prometheus Alert Manager", commando.Bool, true).
+               AddFlag("silenced", "Silenced alerts in Prometheus Alert Manager", commando.Bool, true).
+               AddFlag("unprocessed", "Unprocessed alerts in Prometheus Alert Manager", commando.Bool, true).
+               AddFlag("host", "Prometheus Alert Manager host address", commando.String, nil).
+               AddFlag("port", "Prometheus Alert Manager port", commando.String, "9093").
+               SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) {
+                       displayAlerts(flags)
+               })
+
        // parse command-line arguments
        commando.Parse(nil)
 }
@@ -561,3 +580,90 @@ func raiseClearAlarmOverPeriod(alarmobject *alarm.Alarm, flags map[string]comman
                }
        }
 }
+
+func displayAlerts(flags map[string]commando.FlagValue) {
+       resp, err := getAlerts(flags)
+       if err != nil {
+               fmt.Println(err)
+               return
+       }
+
+       if resp == nil {
+               fmt.Println("resp= nil")
+               return
+       }
+
+       t := table.NewWriter()
+       t.SetOutputMirror(os.Stdout)
+       t.AppendHeader(table.Row{"Alerts from Prometheus Alert Manager"})
+       for _, gettableAlert := range resp.Payload{
+               t.AppendRow([]interface{}{"------------------------------------"})
+               if gettableAlert != nil {
+                       for key, item := range gettableAlert.Annotations {
+                               t.AppendRow([]interface{}{key, item})   
+                       }
+                       if gettableAlert.EndsAt != nil {
+                               t.AppendRow([]interface{}{"EndsAt", *gettableAlert.EndsAt})
+                       }
+                       if gettableAlert.Fingerprint != nil {
+                               t.AppendRow([]interface{}{"Fingerprint", *gettableAlert.Fingerprint})
+                       }
+                       for key, item := range gettableAlert.Receivers {
+                               if gettableAlert.Receivers != nil {
+                                       t.AppendRow([]interface{}{key, *item.Name})     
+                               }
+                       }
+                       if gettableAlert.StartsAt != nil {
+                               t.AppendRow([]interface{}{"StartsAt", *gettableAlert.StartsAt})
+                       }
+                       if gettableAlert.Status != nil {
+                               t.AppendRow([]interface{}{"InhibitedBy", gettableAlert.Status.InhibitedBy})
+                               t.AppendRow([]interface{}{"SilencedBy", gettableAlert.Status.SilencedBy})
+                               t.AppendRow([]interface{}{"State", *gettableAlert.Status.State})
+                       }
+                       if gettableAlert.UpdatedAt != nil {
+                               t.AppendRow([]interface{}{"UpdatedAt", *gettableAlert.UpdatedAt})
+                       }
+                       t.AppendRow([]interface{}{"GeneratorURL", gettableAlert.Alert.GeneratorURL})
+                       for key, item := range gettableAlert.Alert.Labels {
+                               t.AppendRow([]interface{}{key, item})   
+                       }
+               }
+       }
+       t.SetStyle(table.StyleColoredBright)
+       t.Render()
+}
+       
+func getAlerts(flags map[string]commando.FlagValue) (*alert.GetAlertsOK, error) {
+       active, _ := flags["active"].GetBool()
+       inhibited, _ := flags["inhibited"].GetBool()
+       silenced, _ := flags["silenced"].GetBool()
+       unprocessed, _ := flags["unprocessed"].GetBool()
+       amHost, _ := flags["host"].GetString()
+       amPort, _ := flags["port"].GetString()
+       var amAddress string
+       if amHost == "" {
+               amAddress = viper.GetString("controls.promAlertManager.address")
+       } else {
+               amAddress = amHost + ":" + amPort
+       }
+
+       alertParams := alert.NewGetAlertsParams()
+       alertParams.Active = &active
+       alertParams.Inhibited = &inhibited
+       alertParams.Silenced = &silenced
+       alertParams.Unprocessed = &unprocessed
+       amBaseUrl := viper.GetString("controls.promAlertManager.baseUrl")
+       amSchemes := []string{viper.GetString("controls.promAlertManager.schemes")}
+       resp, err := newAlertManagerClient(amAddress, amBaseUrl, amSchemes).Alert.GetAlerts(alertParams)
+       if err != nil {
+               err = fmt.Errorf("GetAlerts from '%s%s' failed with error: %v", amAddress, amBaseUrl, err)
+       }
+       return resp, err
+}
+
+func newAlertManagerClient(amAddress string, amBaseUrl string, amSchemes []string) *client.Alertmanager {
+       cr := clientruntime.New(amAddress, amBaseUrl, amSchemes)
+       return client.New(cr, strfmt.Default)
+}
+
index 2d5b633..e531f7c 100755 (executable)
@@ -151,9 +151,15 @@ CLI commands can have some of the following parameters
  --nal         Number of alarms, example value: 50
  --aps         Alarms per second, example value: 1
  --tim         Total time of test in minutes, example value: 1 
- --host        Alarm Manager REST address: default value = localhost
- --port        Alarm Manager REST port: default value = 8080
+ --host        Alarm Manager host address. Default value = localhost
+ --port        Alarm Manager port. Default value = 8080
  --if          Used Alarm Manager command interface, http or rmr: default value = http
+ --active      Active alerts in Prometheus Alert Manager. Default value = true
+ --inhibited   Inhibited alerts in Prometheus Alert Manager. Default value = true
+ --silenced    Silenced alerts in Prometheus Alert Manager. Default value = true
+ --unprocessed Unprocessed alerts in Prometheus Alert Manager. Default value = true
+ --host        Prometheus Alert Manager host address
+ --port        Prometheus Alert Manager port. Default value = 9093
 
 
 ``Note that there are two minus signs before parameter name!``
@@ -261,6 +267,14 @@ CLI command examples:
 
   Endurance test example: cli/alarm-cli perf --prf 2 --nal 50 --aps 1 --tim 1 --host localhost --port 8080 --if rmr
 
+Get alerts from Prometheus Alert Manager: 
+
+ .. code-block:: none
+
+  Syntax: cli/alarm-cli gapam --active --inhibited --silenced --unprocessed --host [--port]
+
+  Example: cli/alarm-cli gapam --active true --inhibited true --silenced true --unprocessed true --host 10.102.36.121 --port 9093
+
 
 REST interface usage guide
 --------------------------
index 34b3902..a7274c2 100755 (executable)
@@ -35,6 +35,7 @@ import (
        "net/http"
        "net/http/httptest"
        "os"
+       "os/exec"
        "strconv"
        "strings"
        "testing"
@@ -331,6 +332,20 @@ func TestActiveAlarmMaxThresholds(t *testing.T) {
        ts.Close()
 }
 
+func TestGetPrometheusAlerts(t *testing.T) {
+       time.Sleep(1 * time.Second)
+       xapp.Logger.Info("TestGetPrometheusAlerts")
+       ts := CreatePromAlertSimulator2(t, "GET", "/alerts?active=true&inhibited=true&silenced=true&unprocessed=true")
+
+       commandReady := make(chan bool, 1)
+       command := "cli/alarm-cli"
+       args := []string {"gapam", "--active", "true", "--inhibited", "true", "--silenced", "--unprocessed", "true", "true", "--host", "localhost", "--port", "9093", "flushall"}
+       ExecCLICommand(commandReady, command, args...)
+       <-commandReady
+
+       ts.Close()
+}
+
 func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string {
        receivedAlert := waitForEvent()
 
@@ -370,6 +385,33 @@ func CreatePromAlertSimulator(t *testing.T, method, url string, status int, resp
        return ts
 }
 
+func CreatePromAlertSimulator2(t *testing.T, method, url string) *httptest.Server {
+       l, err := net.Listen("tcp", "localhost:9093")
+       if err != nil {
+               t.Error("Failed to create listener: " + err.Error())
+       }
+       ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+               assert.Equal(t, r.Method, method)
+               assert.Equal(t, r.URL.String(), url)
+
+               w.Header().Add("Content-Type", "application/json")
+               w.WriteHeader(200)
+               // Read alerts from file
+               payload, err := readJSONFromFile("../testresources/prometheus-alerts.json")
+                       if err != nil {
+                               t.Error("Failed to send response: ", err)
+               }
+               _, err = w.Write(payload)
+               if err != nil {
+                       t.Error("Failed to send response: " + err.Error())
+               }
+       }))
+       ts.Listener.Close()
+       ts.Listener = l
+       ts.Start()
+       return ts
+}
+
 func waitForEvent() string {
        receivedAlert := <-eventChan
        return receivedAlert
@@ -398,3 +440,27 @@ func checkResponseCode(t *testing.T, expected, actual int) bool {
        }
        return true
 }
+
+func ExecCLICommand(commandReady chan bool, command string, args ...string) {
+       go func() {
+               xapp.Logger.Info("Giving CLI command")
+               cmd := exec.Command(command, args...)
+               cmd.Dir = "../"
+               output, err := cmd.CombinedOutput()
+               if err != nil {
+                       xapp.Logger.Info("CLI command failed out: %s", err)
+               }
+               xapp.Logger.Info("CLI command output: %s", output)
+               commandReady <- true
+               xapp.Logger.Info("CLI command completed")
+       }()
+}
+
+func readJSONFromFile(filename string) ([]byte, error) {
+       file, err := ioutil.ReadFile(filename)
+       if err != nil {
+               err := fmt.Errorf("readJSONFromFile() failed: Error: %v", err)
+               return nil, err
+       }
+       return file, nil
+}
diff --git a/testresources/prometheus-alerts.json b/testresources/prometheus-alerts.json
new file mode 100644 (file)
index 0000000..c3b2e31
--- /dev/null
@@ -0,0 +1,42 @@
+[{
+"annotations":{"alertname":"RIC ROUTING TABLE DISTRIBUTION FAILED",
+"service":"my-pod:my-app",
+"severity":"MAJOR",
+"status":"active",
+"system_name":"RIC:my-pod:my-app"},
+"endsAt":"2019-01-01T11:00:00.000Z",
+"fingerprint":"ABCDEF",
+"receivers":[{"name":"Some name"},{"name":"Another name"}],
+"startsAt":"2019-01-01T12:00:00.000Z",
+"status":{"inhibitedBy":[""],
+"silencedBy":[""],
+"state":"state"},
+"updatedAt":"2019-01-01T12:00:00.000Z",
+"labels":{"Additional_info":"-",
+"AlarmId":"8004",
+"Description":"004:eth 0 1:Some App data",
+"Instructions":"Not defined",
+"Summary":"Processing error",
+"Timestamp":"15/10/2020, 15:09:14"           }
+},
+{
+"annotations":{"alertname":"TCP CONNECTIVITY LOST TO DBAAS",
+"service":"my-pod:my-app",
+"severity":"MAJOR",
+"status":"active",
+"system_name":"RIC:my-pod:my-app"},
+"endsAt":"2019-01-01T11:00:00.000Z",
+"fingerprint":"ABCDEF",
+"receivers":[{"name":"Some name"}],
+"startsAt":"2019-01-01T12:00:00.000Z",
+"status":{"inhibitedBy":["Inhibitor_1", "Inhibitor_2"],
+"silencedBy":["Silencer_1", "Silencer_2"],
+"state":"state"},
+"updatedAt":"2019-01-01T12:00:00.000Z",
+"labels":{"Additional_info":"-",
+"AlarmId":"8005",
+"Description":"004:eth 0 1:Some App data",
+"Instructions":"Not defined",
+"Summary":"Communication error",
+"Timestamp":"15/10/2020, 15:09:14"           }
+}]
\ No newline at end of file