From 00894a4120ea2c93bc58327fbb7c526a6f607bde Mon Sep 17 00:00:00 2001 From: Anssi Mannila Date: Mon, 19 Oct 2020 11:36:26 +0300 Subject: [PATCH] LN0739_FM_RF13: support interface to external Prometheus alert manager Change-Id: Ia255950c07f3abc51edbe139f4856cfd6c03448f Signed-off-by: Anssi Mannila --- build/Dockerfile | 4 +- build/build_ubuntu.sh | 11 ++- build/run_local.sh | 4 +- cli/alarm-cli.go | 108 ++++++++++++++++++++- docs/user-guide.rst | 18 +++- manager/cmd/manager_test.go | 66 +++++++++++++ .../perf-alarm-definition.json | 0 .../perf-alarm-object.json | 0 testresources/prometheus-alerts.json | 42 ++++++++ 9 files changed, 241 insertions(+), 12 deletions(-) rename {perfresources => testresources}/perf-alarm-definition.json (100%) rename {perfresources => testresources}/perf-alarm-object.json (100%) create mode 100644 testresources/prometheus-alerts.json diff --git a/build/Dockerfile b/build/Dockerfile index 7e9ebc8..b2d0884 100755 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -36,8 +36,8 @@ FROM ubuntu:18.04 COPY --from=ubuntu-alarm-manager /go/src/am/build/run.sh / COPY --from=ubuntu-alarm-manager /go/src/am/manager/alarm-manager / COPY --from=ubuntu-alarm-manager /go/src/am/cli/alarm-cli / -COPY --from=ubuntu-alarm-manager /go/src/am/perfresources/perf-alarm-object.json / -COPY --from=ubuntu-alarm-manager /go/src/am/perfresources/perf-alarm-definition.json / +COPY --from=ubuntu-alarm-manager /go/src/am/testresources/perf-alarm-object.json / +COPY --from=ubuntu-alarm-manager /go/src/am/testresources/perf-alarm-definition.json / COPY --from=ubuntu-alarm-manager /go/src/am/config/* / COPY --from=ubuntu-alarm-manager /usr/local/lib /usr/local/lib COPY --from=ubuntu-alarm-manager /go/src/am/definitions/* / diff --git a/build/build_ubuntu.sh b/build/build_ubuntu.sh index 79e5a8c..26bc0e3 100755 --- a/build/build_ubuntu.sh +++ b/build/build_ubuntu.sh @@ -44,8 +44,8 @@ export RMR_SEED_RT=../config/uta_rtg.rt # xApp stuff export DEF_FILE=../definitions/alarm-definition.json -export PERF_DEF_FILE=../perfresources/perf-alarm-definition.json -export PERF_OBJ_FILE=../perfresources/perf-alarm-object.json +export PERF_DEF_FILE=../testresources/perf-alarm-definition.json +export PERF_OBJ_FILE=../testresources/perf-alarm-object.json GO111MODULE=on GO_ENABLED=0 GOOS=linux @@ -61,6 +61,10 @@ hash=$(git rev-parse --short HEAD || true) ROOT_DIR=$PWD + +# compile the CLI +cd ${ROOT_DIR}/cli && go build -a -installsuffix cgo alarm-cli.go + # Build cd ${ROOT_DIR}/manager && go build -a -installsuffix cgo -ldflags "-X main.Version=$tag -X main.Hash=$hash" -o alarm-manager ./cmd/*.go @@ -70,7 +74,4 @@ cd ${ROOT_DIR}/alarm && go test . -v -coverprofile cover.out # And for the Alarm Manager cd ${ROOT_DIR}/manager && go test -v -p 1 -coverprofile cover.out ./cmd/ -c -o ./manager_test && ./manager_test -# Finally compile the CLI -cd ${ROOT_DIR}/cli && go build -a -installsuffix cgo alarm-cli.go - echo "--> build_ubuntu.sh ends" diff --git a/build/run_local.sh b/build/run_local.sh index ddd4524..24fdd1a 100755 --- a/build/run_local.sh +++ b/build/run_local.sh @@ -25,7 +25,7 @@ export RMR_SEED_RT=$PWD/config/uta_rtg.rt export RMR_SRC_ID="service-ricplt-alarmmanager-rmr.ricplt" export DEF_FILE=$PWD/definitions/alarm-definition.json -export PERF_DEF_FILE=$PWD/perfresources/perf-alarm-definition.json -export PERF_OBJ_FILE=$PWD/perfresources/perf-alarm-object.json +export PERF_DEF_FILE=$PWD/testresources/perf-alarm-definition.json +export PERF_OBJ_FILE=$PWD/testresources/perf-alarm-object.json $PWD/manager/alarm-manager -f $PWD/config/config-file.json diff --git a/cli/alarm-cli.go b/cli/alarm-cli.go index afe9da9..f666292 100755 --- a/cli/alarm-cli.go +++ b/cli/alarm-cli.go @@ -8,12 +8,17 @@ import ( "net/http" "os" "strconv" + "sync" "time" "gerrit.o-ran-sc.org/r/ric-plt/alarm-go/alarm" + clientruntime "github.com/go-openapi/runtime/client" + "github.com/go-openapi/strfmt" "github.com/jedib0t/go-pretty/table" + "github.com/prometheus/alertmanager/api/v2/client" + "github.com/prometheus/alertmanager/api/v2/client/alert" "github.com/thatisuday/commando" - "sync" + "github.com/spf13/viper" ) type CliAlarmDefinitions struct { @@ -153,6 +158,20 @@ func main() { conductperformancetest(flags) }) + // Get alerts from Prometheus Alert Manager + commando. + Register("gapam"). + SetShortDescription("Get alerts from Prometheus Alert Manager"). + AddFlag("active", "Active alerts in Prometheus Alert Manager", commando.Bool, true). + AddFlag("inhibited", "Inhibited alerts in Prometheus Alert Manager", commando.Bool, true). + AddFlag("silenced", "Silenced alerts in Prometheus Alert Manager", commando.Bool, true). + AddFlag("unprocessed", "Unprocessed alerts in Prometheus Alert Manager", commando.Bool, true). + AddFlag("host", "Prometheus Alert Manager host address", commando.String, nil). + AddFlag("port", "Prometheus Alert Manager port", commando.String, "9093"). + SetAction(func(args map[string]commando.ArgValue, flags map[string]commando.FlagValue) { + displayAlerts(flags) + }) + // parse command-line arguments commando.Parse(nil) } @@ -561,3 +580,90 @@ func raiseClearAlarmOverPeriod(alarmobject *alarm.Alarm, flags map[string]comman } } } + +func displayAlerts(flags map[string]commando.FlagValue) { + resp, err := getAlerts(flags) + if err != nil { + fmt.Println(err) + return + } + + if resp == nil { + fmt.Println("resp= nil") + return + } + + t := table.NewWriter() + t.SetOutputMirror(os.Stdout) + t.AppendHeader(table.Row{"Alerts from Prometheus Alert Manager"}) + for _, gettableAlert := range resp.Payload{ + t.AppendRow([]interface{}{"------------------------------------"}) + if gettableAlert != nil { + for key, item := range gettableAlert.Annotations { + t.AppendRow([]interface{}{key, item}) + } + if gettableAlert.EndsAt != nil { + t.AppendRow([]interface{}{"EndsAt", *gettableAlert.EndsAt}) + } + if gettableAlert.Fingerprint != nil { + t.AppendRow([]interface{}{"Fingerprint", *gettableAlert.Fingerprint}) + } + for key, item := range gettableAlert.Receivers { + if gettableAlert.Receivers != nil { + t.AppendRow([]interface{}{key, *item.Name}) + } + } + if gettableAlert.StartsAt != nil { + t.AppendRow([]interface{}{"StartsAt", *gettableAlert.StartsAt}) + } + if gettableAlert.Status != nil { + t.AppendRow([]interface{}{"InhibitedBy", gettableAlert.Status.InhibitedBy}) + t.AppendRow([]interface{}{"SilencedBy", gettableAlert.Status.SilencedBy}) + t.AppendRow([]interface{}{"State", *gettableAlert.Status.State}) + } + if gettableAlert.UpdatedAt != nil { + t.AppendRow([]interface{}{"UpdatedAt", *gettableAlert.UpdatedAt}) + } + t.AppendRow([]interface{}{"GeneratorURL", gettableAlert.Alert.GeneratorURL}) + for key, item := range gettableAlert.Alert.Labels { + t.AppendRow([]interface{}{key, item}) + } + } + } + t.SetStyle(table.StyleColoredBright) + t.Render() +} + +func getAlerts(flags map[string]commando.FlagValue) (*alert.GetAlertsOK, error) { + active, _ := flags["active"].GetBool() + inhibited, _ := flags["inhibited"].GetBool() + silenced, _ := flags["silenced"].GetBool() + unprocessed, _ := flags["unprocessed"].GetBool() + amHost, _ := flags["host"].GetString() + amPort, _ := flags["port"].GetString() + var amAddress string + if amHost == "" { + amAddress = viper.GetString("controls.promAlertManager.address") + } else { + amAddress = amHost + ":" + amPort + } + + alertParams := alert.NewGetAlertsParams() + alertParams.Active = &active + alertParams.Inhibited = &inhibited + alertParams.Silenced = &silenced + alertParams.Unprocessed = &unprocessed + amBaseUrl := viper.GetString("controls.promAlertManager.baseUrl") + amSchemes := []string{viper.GetString("controls.promAlertManager.schemes")} + resp, err := newAlertManagerClient(amAddress, amBaseUrl, amSchemes).Alert.GetAlerts(alertParams) + if err != nil { + err = fmt.Errorf("GetAlerts from '%s%s' failed with error: %v", amAddress, amBaseUrl, err) + } + return resp, err +} + +func newAlertManagerClient(amAddress string, amBaseUrl string, amSchemes []string) *client.Alertmanager { + cr := clientruntime.New(amAddress, amBaseUrl, amSchemes) + return client.New(cr, strfmt.Default) +} + diff --git a/docs/user-guide.rst b/docs/user-guide.rst index 2d5b633..e531f7c 100755 --- a/docs/user-guide.rst +++ b/docs/user-guide.rst @@ -151,9 +151,15 @@ CLI commands can have some of the following parameters --nal Number of alarms, example value: 50 --aps Alarms per second, example value: 1 --tim Total time of test in minutes, example value: 1 - --host Alarm Manager REST address: default value = localhost - --port Alarm Manager REST port: default value = 8080 + --host Alarm Manager host address. Default value = localhost + --port Alarm Manager port. Default value = 8080 --if Used Alarm Manager command interface, http or rmr: default value = http + --active Active alerts in Prometheus Alert Manager. Default value = true + --inhibited Inhibited alerts in Prometheus Alert Manager. Default value = true + --silenced Silenced alerts in Prometheus Alert Manager. Default value = true + --unprocessed Unprocessed alerts in Prometheus Alert Manager. Default value = true + --host Prometheus Alert Manager host address + --port Prometheus Alert Manager port. Default value = 9093 ``Note that there are two minus signs before parameter name!`` @@ -261,6 +267,14 @@ CLI command examples: Endurance test example: cli/alarm-cli perf --prf 2 --nal 50 --aps 1 --tim 1 --host localhost --port 8080 --if rmr +Get alerts from Prometheus Alert Manager: + + .. code-block:: none + + Syntax: cli/alarm-cli gapam --active --inhibited --silenced --unprocessed --host [--port] + + Example: cli/alarm-cli gapam --active true --inhibited true --silenced true --unprocessed true --host 10.102.36.121 --port 9093 + REST interface usage guide -------------------------- diff --git a/manager/cmd/manager_test.go b/manager/cmd/manager_test.go index 34b3902..a7274c2 100755 --- a/manager/cmd/manager_test.go +++ b/manager/cmd/manager_test.go @@ -35,6 +35,7 @@ import ( "net/http" "net/http/httptest" "os" + "os/exec" "strconv" "strings" "testing" @@ -331,6 +332,20 @@ func TestActiveAlarmMaxThresholds(t *testing.T) { ts.Close() } +func TestGetPrometheusAlerts(t *testing.T) { + time.Sleep(1 * time.Second) + xapp.Logger.Info("TestGetPrometheusAlerts") + ts := CreatePromAlertSimulator2(t, "GET", "/alerts?active=true&inhibited=true&silenced=true&unprocessed=true") + + commandReady := make(chan bool, 1) + command := "cli/alarm-cli" + args := []string {"gapam", "--active", "true", "--inhibited", "true", "--silenced", "--unprocessed", "true", "true", "--host", "localhost", "--port", "9093", "flushall"} + ExecCLICommand(commandReady, command, args...) + <-commandReady + + ts.Close() +} + func VerifyAlarm(t *testing.T, a alarm.Alarm, expectedCount int) string { receivedAlert := waitForEvent() @@ -370,6 +385,33 @@ func CreatePromAlertSimulator(t *testing.T, method, url string, status int, resp return ts } +func CreatePromAlertSimulator2(t *testing.T, method, url string) *httptest.Server { + l, err := net.Listen("tcp", "localhost:9093") + if err != nil { + t.Error("Failed to create listener: " + err.Error()) + } + ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, r.Method, method) + assert.Equal(t, r.URL.String(), url) + + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(200) + // Read alerts from file + payload, err := readJSONFromFile("../testresources/prometheus-alerts.json") + if err != nil { + t.Error("Failed to send response: ", err) + } + _, err = w.Write(payload) + if err != nil { + t.Error("Failed to send response: " + err.Error()) + } + })) + ts.Listener.Close() + ts.Listener = l + ts.Start() + return ts +} + func waitForEvent() string { receivedAlert := <-eventChan return receivedAlert @@ -398,3 +440,27 @@ func checkResponseCode(t *testing.T, expected, actual int) bool { } return true } + +func ExecCLICommand(commandReady chan bool, command string, args ...string) { + go func() { + xapp.Logger.Info("Giving CLI command") + cmd := exec.Command(command, args...) + cmd.Dir = "../" + output, err := cmd.CombinedOutput() + if err != nil { + xapp.Logger.Info("CLI command failed out: %s", err) + } + xapp.Logger.Info("CLI command output: %s", output) + commandReady <- true + xapp.Logger.Info("CLI command completed") + }() +} + +func readJSONFromFile(filename string) ([]byte, error) { + file, err := ioutil.ReadFile(filename) + if err != nil { + err := fmt.Errorf("readJSONFromFile() failed: Error: %v", err) + return nil, err + } + return file, nil +} diff --git a/perfresources/perf-alarm-definition.json b/testresources/perf-alarm-definition.json similarity index 100% rename from perfresources/perf-alarm-definition.json rename to testresources/perf-alarm-definition.json diff --git a/perfresources/perf-alarm-object.json b/testresources/perf-alarm-object.json similarity index 100% rename from perfresources/perf-alarm-object.json rename to testresources/perf-alarm-object.json diff --git a/testresources/prometheus-alerts.json b/testresources/prometheus-alerts.json new file mode 100644 index 0000000..c3b2e31 --- /dev/null +++ b/testresources/prometheus-alerts.json @@ -0,0 +1,42 @@ +[{ +"annotations":{"alertname":"RIC ROUTING TABLE DISTRIBUTION FAILED", +"service":"my-pod:my-app", +"severity":"MAJOR", +"status":"active", +"system_name":"RIC:my-pod:my-app"}, +"endsAt":"2019-01-01T11:00:00.000Z", +"fingerprint":"ABCDEF", +"receivers":[{"name":"Some name"},{"name":"Another name"}], +"startsAt":"2019-01-01T12:00:00.000Z", +"status":{"inhibitedBy":[""], +"silencedBy":[""], +"state":"state"}, +"updatedAt":"2019-01-01T12:00:00.000Z", +"labels":{"Additional_info":"-", +"AlarmId":"8004", +"Description":"004:eth 0 1:Some App data", +"Instructions":"Not defined", +"Summary":"Processing error", +"Timestamp":"15/10/2020, 15:09:14" } +}, +{ +"annotations":{"alertname":"TCP CONNECTIVITY LOST TO DBAAS", +"service":"my-pod:my-app", +"severity":"MAJOR", +"status":"active", +"system_name":"RIC:my-pod:my-app"}, +"endsAt":"2019-01-01T11:00:00.000Z", +"fingerprint":"ABCDEF", +"receivers":[{"name":"Some name"}], +"startsAt":"2019-01-01T12:00:00.000Z", +"status":{"inhibitedBy":["Inhibitor_1", "Inhibitor_2"], +"silencedBy":["Silencer_1", "Silencer_2"], +"state":"state"}, +"updatedAt":"2019-01-01T12:00:00.000Z", +"labels":{"Additional_info":"-", +"AlarmId":"8005", +"Description":"004:eth 0 1:Some App data", +"Instructions":"Not defined", +"Summary":"Communication error", +"Timestamp":"15/10/2020, 15:09:14" } +}] \ No newline at end of file -- 2.16.6