From 7ca707c2385d69914ea6911a8c22537faf1fd74f Mon Sep 17 00:00:00 2001 From: Arif Hussain Date: Thu, 2 Dec 2021 15:20:40 +0530 Subject: [PATCH] Add HTTPS support for VES events This task to add support for the secure form of HTTP for VES events. To begin with only the server identity will be checked as part of sending a POST to the VES collector. Issue-Id: SMO-17 Signed-off-by: Arif Hussain Change-Id: I9923ed00168ea8af4f7f500e80101aa2625e7c5d Signed-off-by: Arif Hussain --- README | 12 ++++++++++++ agent/Dockerfile | 1 + .../3rd_party/collectd-ves-app/ves_app/ves_app.py | 18 ++++++++++-------- collector/Dockerfile | 4 ++-- .../evel-test-collector/code/collector/monitor.py | 3 ++- docker-compose.yaml | 4 +++- 6 files changed, 30 insertions(+), 12 deletions(-) diff --git a/README b/README index e6f3273..5a63a19 100644 --- a/README +++ b/README @@ -22,3 +22,15 @@ RUN: There are two scripts in this folder. A ves-start.sh script which starts the VES collector and other parts. A ves-stop.sh script can be used to stop the collector. + + +Following steps are required for self-signed certificate. + # Create ves-certificate directory on the host system using command "mkdir ~/ves-certificate". + # Go to ves-certificate directory and use below commands to create self-signed certificate files. + + openssl genrsa -out vescertificate.key 2048 + openssl req -new -key vescertificate.key -out vescertificate.csr + openssl x509 -req -days 365 -in vescertificate.csr -signkey vescertificate.key -out vescertificate.crt + +**Note**: Third party certificates can be installed by overwriting the file vescertificate.csr, vescertificate.key, and vescertficate.crt in ~/ves-certificate directory of the host system. + diff --git a/agent/Dockerfile b/agent/Dockerfile index 238d877..82bbf3e 100755 --- a/agent/Dockerfile +++ b/agent/Dockerfile @@ -34,6 +34,7 @@ RUN apt-get install -y default-jre zookeeperd \ python3 python3-pip pkg-config git build-essential libpthread-stubs0-dev \ libssl-dev libsasl2-dev liblz4-dev libz-dev RUN pip3 install kafka-python pyaml +RUN pip3 install --upgrade certifi RUN mkdir /opt/ves/barometer ADD barometer /opt/ves/barometer diff --git a/agent/barometer/3rd_party/collectd-ves-app/ves_app/ves_app.py b/agent/barometer/3rd_party/collectd-ves-app/ves_app/ves_app.py index a519d2f..71aa1c6 100755 --- a/agent/barometer/3rd_party/collectd-ves-app/ves_app/ves_app.py +++ b/agent/barometer/3rd_party/collectd-ves-app/ves_app/ves_app.py @@ -20,6 +20,8 @@ import sys import base64 import logging import argparse +import ssl +from socket import timeout try: import configparser @@ -80,16 +82,16 @@ class VESApp(Normalizer): request.add_header('Content-Type', 'application/json') event_str = json.dumps(event).encode() logging.debug("Sending {} to {}".format(event_str, server_url)) - url.urlopen(request, event_str, timeout=1) + ssl._create_default_https_context = ssl._create_unverified_context + url.urlopen(request, event_str, timeout=1).read().decode('utf-8') logging.debug("Sent data to {} successfully".format(server_url)) - except url.HTTPError as e: - logging.error('Vendor Event Listener exception: {}'.format(e)) - except url.URLError as e: - logging.error( - 'Vendor Event Listener is is not reachable: {}'.format(e)) + except (HTTPError, URLError) as e: + logging.error('Vendor Event Listener is is not reachable: {}'.format(e)) + except timeout: + logging.error('Timed out - URL %s', url) except Exception as e: - logging.error('Vendor Event Listener error: {}'.format(e)) - + logging.error('Vendor Event Listener error: {}'.format(e)) + def config(self, config): """VES option configuration""" for key, value in config.items('config'): diff --git a/collector/Dockerfile b/collector/Dockerfile index f69561b..c70184e 100755 --- a/collector/Dockerfile +++ b/collector/Dockerfile @@ -22,10 +22,10 @@ FROM ubuntu:focal RUN apt-get update && apt-get -y upgrade RUN apt-get install -y git curl python3 python3-pip -RUN pip3 install requests jsonschema elasticsearch kafka-python +RUN pip3 install requests jsonschema elasticsearch kafka-python gevent -RUN mkdir /opt/ves +RUN mkdir -p /opt/ves/certs # Clone VES Collector RUN mkdir /opt/ves/evel-test-collector diff --git a/collector/evel-test-collector/code/collector/monitor.py b/collector/evel-test-collector/code/collector/monitor.py index ae9de29..784139a 100755 --- a/collector/evel-test-collector/code/collector/monitor.py +++ b/collector/evel-test-collector/code/collector/monitor.py @@ -40,6 +40,7 @@ from kafka import KafkaProducer from json import dumps import datetime import time +from gevent import pywsgi monitor_mode = "f" vdu_id = ['', '', '', '', '', ''] @@ -1026,7 +1027,7 @@ USAGE dispatcher.register('POST', test_control_url, test_control_listener) dispatcher.register('GET', test_control_url, test_control_listener) - httpd = make_server('', int(vel_port), vendor_event_listener) + httpd = pywsgi.WSGIServer(('', int(vel_port)), vendor_event_listener, keyfile='/opt/ves/certs/vescertificate.key', certfile='/opt/ves/certs/vescertificate.crt') logger.info('Serving on port {0}...'.format(vel_port)) httpd.serve_forever() diff --git a/docker-compose.yaml b/docker-compose.yaml index 6a03f2c..4f5bbaf 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -107,6 +107,8 @@ services: - ves-net ports: - 9999:9999 + volumes: + - ~/ves-certificate:/opt/ves/certs environment: ves_influxdb_host: "ves-influxdb" ves_influxdb_port: "8086" @@ -144,7 +146,7 @@ services: ves_port: "9999" ves_path: "" ves_topic: "events" - ves_https: "False" + ves_https: "True" ves_user: "user" ves_pass: "password" ves_interval: "10" -- 2.16.6