Add HTTPS support for VES events

This task to add support for the secure form of HTTP for VES events. To begin with only the server identity will be checked as part of sending a POST to the VES collector.

Issue-Id: SMO-17

Signed-off-by: Arif Hussain <arif.hussain@xoriant.com>
Change-Id: I9923ed00168ea8af4f7f500e80101aa2625e7c5d
Signed-off-by: Arif Hussain <arif.hussain@xoriant.com>

diff --git a/README b/README
index e6f3273..5a63a19 100644 (file)
--- a/README
+++ b/README
@@ -22,3 +22,15 @@ RUN:
 There are two scripts in this folder. A ves-start.sh script
 which starts the VES collector and other parts. A ves-stop.sh script
 can be used to stop the collector.
+
+
+Following steps are required for self-signed certificate.
+ # Create ves-certificate directory on the host system using command "mkdir ~/ves-certificate".
+ # Go to ves-certificate directory and use below commands to create self-signed certificate files.
+
+               openssl genrsa -out vescertificate.key 2048
+               openssl req -new -key vescertificate.key -out vescertificate.csr
+               openssl x509 -req -days 365 -in vescertificate.csr -signkey vescertificate.key -out vescertificate.crt
+
+**Note**: Third party certificates can be installed by overwriting the file vescertificate.csr, vescertificate.key, and vescertficate.crt in ~/ves-certificate directory of the host system.
+

diff --git a/agent/Dockerfile b/agent/Dockerfile
index 238d877..82bbf3e 100755 (executable)
--- a/agent/Dockerfile
+++ b/agent/Dockerfile
@@ -34,6 +34,7 @@ RUN apt-get install -y default-jre zookeeperd \
 python3 python3-pip pkg-config git build-essential libpthread-stubs0-dev \
 libssl-dev libsasl2-dev liblz4-dev libz-dev
 RUN pip3 install kafka-python pyaml
+RUN pip3 install --upgrade certifi
 
 RUN mkdir /opt/ves/barometer
 ADD barometer /opt/ves/barometer

diff --git a/agent/barometer/3rd_party/collectd-ves-app/ves_app/ves_app.py b/agent/barometer/3rd_party/collectd-ves-app/ves_app/ves_app.py
index a519d2f..71aa1c6 100755 (executable)
--- a/agent/barometer/3rd_party/collectd-ves-app/ves_app/ves_app.py
+++ b/agent/barometer/3rd_party/collectd-ves-app/ves_app/ves_app.py
@@ -20,6 +20,8 @@ import sys
 import base64
 import logging
 import argparse
+import ssl
+from socket import timeout
 
 try:
     import configparser
@@ -80,16 +82,16 @@ class VESApp(Normalizer):
             request.add_header('Content-Type', 'application/json')
             event_str = json.dumps(event).encode()
             logging.debug("Sending {} to {}".format(event_str, server_url))
-            url.urlopen(request, event_str, timeout=1)
+            ssl._create_default_https_context = ssl._create_unverified_context
+            url.urlopen(request, event_str, timeout=1).read().decode('utf-8')
             logging.debug("Sent data to {} successfully".format(server_url))
-        except url.HTTPError as e:
-            logging.error('Vendor Event Listener exception: {}'.format(e))
-        except url.URLError as e:
-            logging.error(
-                'Vendor Event Listener is is not reachable: {}'.format(e))
+        except (HTTPError, URLError) as e:
+            logging.error('Vendor Event Listener is is not reachable: {}'.format(e))
+        except timeout:
+            logging.error('Timed out - URL %s', url)
         except Exception as e:
-            logging.error('Vendor Event Listener error: {}'.format(e))
-
+            logging.error('Vendor Event Listener error: {}'.format(e)) 
+            
     def config(self, config):
         """VES option configuration"""
         for key, value in config.items('config'):

diff --git a/collector/Dockerfile b/collector/Dockerfile
index f69561b..c70184e 100755 (executable)
--- a/collector/Dockerfile
+++ b/collector/Dockerfile
@@ -22,10 +22,10 @@ FROM ubuntu:focal
 
 RUN apt-get update && apt-get -y upgrade
 RUN apt-get install -y git curl python3 python3-pip
-RUN pip3 install requests jsonschema elasticsearch kafka-python
+RUN pip3 install requests jsonschema elasticsearch kafka-python gevent
 
 
-RUN mkdir /opt/ves
+RUN mkdir -p /opt/ves/certs
 
 # Clone VES Collector
 RUN mkdir /opt/ves/evel-test-collector

diff --git a/collector/evel-test-collector/code/collector/monitor.py b/collector/evel-test-collector/code/collector/monitor.py
index ae9de29..784139a 100755 (executable)
--- a/collector/evel-test-collector/code/collector/monitor.py
+++ b/collector/evel-test-collector/code/collector/monitor.py
@@ -40,6 +40,7 @@ from kafka import KafkaProducer
 from json import dumps
 import datetime
 import time
+from gevent import pywsgi
 
 monitor_mode = "f"
 vdu_id = ['', '', '', '', '', '']
@@ -1026,7 +1027,7 @@ USAGE
         dispatcher.register('POST', test_control_url, test_control_listener)
         dispatcher.register('GET', test_control_url, test_control_listener)
 
-        httpd = make_server('', int(vel_port), vendor_event_listener)
+        httpd = pywsgi.WSGIServer(('', int(vel_port)), vendor_event_listener, keyfile='/opt/ves/certs/vescertificate.key', certfile='/opt/ves/certs/vescertificate.crt')
         logger.info('Serving on port {0}...'.format(vel_port))
         httpd.serve_forever()
 

diff --git a/docker-compose.yaml b/docker-compose.yaml
index 6a03f2c..4f5bbaf 100644 (file)
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -107,6 +107,8 @@ services:
              - ves-net
          ports:
              - 9999:9999
+         volumes:
+             - ~/ves-certificate:/opt/ves/certs
          environment:
                  ves_influxdb_host: "ves-influxdb"
                  ves_influxdb_port: "8086"
@@ -144,7 +146,7 @@ services:
                   ves_port: "9999"
                   ves_path: ""
                   ves_topic: "events"
-                  ves_https: "False"
+                  ves_https: "True"
                   ves_user: "user"
                   ves_pass: "password"
                   ves_interval: "10"