From 12ca76f9812946b4943e70e9c56d5d454fe46639 Mon Sep 17 00:00:00 2001 From: josephthaliath Date: Wed, 17 May 2023 15:33:25 +0530 Subject: [PATCH] Scripts to push qoe usecase data for DME integration demo Issue-Id: AIMLFW-45 Change-Id: Id34d1e8e40d7880b1cc3250e2e949dadd13ab13e Signed-off-by: josephthaliath --- demos/hrelease/scripts/get_access_tokens.sh | 23 +++ .../scripts/kafka-client-send-file-ready-qoe.sh | 34 ++++ .../hrelease/scripts/prepare_env_aimlfw_access.sh | 22 +++ .../scripts/push-to-file-ready-topic-qoe.sh | 33 ++++ demos/hrelease/scripts/push_qoe_data.sh | 33 ++++ demos/hrelease/scripts/qoedatapush.py | 181 +++++++++++++++++++++ 6 files changed, 326 insertions(+) create mode 100755 demos/hrelease/scripts/get_access_tokens.sh create mode 100755 demos/hrelease/scripts/kafka-client-send-file-ready-qoe.sh create mode 100755 demos/hrelease/scripts/prepare_env_aimlfw_access.sh create mode 100755 demos/hrelease/scripts/push-to-file-ready-topic-qoe.sh create mode 100755 demos/hrelease/scripts/push_qoe_data.sh create mode 100644 demos/hrelease/scripts/qoedatapush.py diff --git a/demos/hrelease/scripts/get_access_tokens.sh b/demos/hrelease/scripts/get_access_tokens.sh new file mode 100755 index 0000000..49c42ab --- /dev/null +++ b/demos/hrelease/scripts/get_access_tokens.sh @@ -0,0 +1,23 @@ +# ================================================================================== +# +# Copyright (c) 2023 Samsung Electronics Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ================================================================================== + +#!/bin/bash + +INFLUXDB2_TOKEN=$(kubectl exec -n nonrtric influxdb2-0 -c influxdb2 -- influx config ls --json | jq -r .default.token) +echo "Influx DB token: " +echo "$INFLUXDB2_TOKEN" diff --git a/demos/hrelease/scripts/kafka-client-send-file-ready-qoe.sh b/demos/hrelease/scripts/kafka-client-send-file-ready-qoe.sh new file mode 100755 index 0000000..7fca0f0 --- /dev/null +++ b/demos/hrelease/scripts/kafka-client-send-file-ready-qoe.sh @@ -0,0 +1,34 @@ +# ================================================================================== +# +# Copyright (c) 2023 Samsung Electronics Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ================================================================================== + +#!/bin/bash + +NODE_NAME_BASE=$1 +FILENAME=$2 +START_EPOCH=$3 +END_EPOCH=$4 +SRV_ID=0 +SRV="pm-https-server-$SRV_ID.pm-https-server.ran" +HTTPS_PORT=443 +URL="https://$SRV:$HTTPS_PORT/files/$FILENAME" + +EVT='{"event":{"commonEventHeader":{"sequence":0,"eventName":"FileReady","sourceName":"'$NODE_NAME_BASE'","lastEpochMicrosec":'$END_EPOCH',"startEpochMicrosec":'$START_EPOCH',"timeZoneOffset":"UTC+05:00","changeIdentifier":"PM_MEAS_FILES"},"notificationFields":{"notificationFieldsVersion":"notificationFieldsVersion","changeType":"FileReady","changeIdentifier":"PM_MEAS_FILES","arrayOfNamedHashMap":[{"name":"'$FILENAME'","hashMap":{"fileFormatType":"org.3GPP.32.435#measCollec","location":"'$URL'","fileFormatVersion":"V10","compression":"gzip"}}]}}}' +echo $EVT +echo $EVT >> .out.json +cat .out.json | kafka-console-producer --topic file-ready --broker-list kafka-1-kafka-bootstrap.nonrtric:9092 +echo "done" diff --git a/demos/hrelease/scripts/prepare_env_aimlfw_access.sh b/demos/hrelease/scripts/prepare_env_aimlfw_access.sh new file mode 100755 index 0000000..1870764 --- /dev/null +++ b/demos/hrelease/scripts/prepare_env_aimlfw_access.sh @@ -0,0 +1,22 @@ +# ================================================================================== +# +# Copyright (c) 2023 Samsung Electronics Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ================================================================================== + +#!/bin/bash + +kubectl delete RequestAuthentication ics-ra -n nonrtric +kubectl delete AuthorizationPolicy ics-ap -n nonrtric diff --git a/demos/hrelease/scripts/push-to-file-ready-topic-qoe.sh b/demos/hrelease/scripts/push-to-file-ready-topic-qoe.sh new file mode 100755 index 0000000..00c24c1 --- /dev/null +++ b/demos/hrelease/scripts/push-to-file-ready-topic-qoe.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# ================================================================================== +# +# Copyright (c) 2023 Samsung Electronics Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ================================================================================== + + +NODE_NAME_BASE=$1 +FILENAME=$2 +START_EPOCH=$3 +END_EPOCH=$4 + +chmod +x kafka-client-send-file-ready-qoe.sh +kubectl cp kafka-client-send-file-ready-qoe.sh nonrtric/client:/home/appuser -c client + +kubectl exec client -c client -n nonrtric -- bash -c './kafka-client-send-file-ready-qoe.sh '$NODE_NAME_BASE' '$FILENAME' '$START_EPOCH' '$END_EPOCH' ' + +echo "done" + diff --git a/demos/hrelease/scripts/push_qoe_data.sh b/demos/hrelease/scripts/push_qoe_data.sh new file mode 100755 index 0000000..9708715 --- /dev/null +++ b/demos/hrelease/scripts/push_qoe_data.sh @@ -0,0 +1,33 @@ +# ================================================================================== +# +# Copyright (c) 2023 Samsung Electronics Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ================================================================================== + +#!/bin/bash + +if [ $# -lt 3 ]; then + echo "Give all input parameters, e.g ./push_qoe_data.sh " + exit 1 +fi + +kubectl patch StatefulSet pm-https-server -n ran -p '{"spec":{"template":{"spec":{"containers":[{"name":"pm-https-server", "env":[{"name":"ALWAYS_RETURN", "value":""}]}]}}}}' +sleep 20 +while [[ $(kubectl get pods pm-https-server-0 -n ran -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do echo "waiting for pm-https-server to be up" && sleep 1; done +kubectl exec -it pm-https-server-0 -n ran -- mkdir -p /files +sudo apt install python3-pip +pip3 install pandas +wget https://raw.githubusercontent.com/o-ran-sc/ric-app-qp/g-release/src/cells.csv -O qoedata.csv +python3 qoedatapush.py $1 $2 $3 diff --git a/demos/hrelease/scripts/qoedatapush.py b/demos/hrelease/scripts/qoedatapush.py new file mode 100644 index 0000000..ac7792b --- /dev/null +++ b/demos/hrelease/scripts/qoedatapush.py @@ -0,0 +1,181 @@ +# ================================================================================== +# +# Copyright (c) 2023 Samsung Electronics Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ================================================================================== + + + +from xml.dom import minidom, Node +import pandas as pd +import os +import datetime +import subprocess +import gzip +import sys + +#Function to create pm file based on qoe data file +def create_xml_document(measurement_list, row, dir_name, sourcename,index): + + time = row['measTimeStampRf'] + du = str(row['du-id']) + cell = row['nrCellIdentity'].replace('/','_') + print(time,du,cell) + doc = minidom.Document() + + measCollecFile = doc.createElement('measCollecFile') + measCollecFile.setAttribute("xmlns","http://www.3gpp.org/ftp/specs/archive/32_series/32.435#measCollec") + measCollecFile.setAttribute("xmlns:xsi","http://www.w3.org/2001/XMLSchema-instance") + measCollecFile.setAttribute("xsi:schemaLocation","http://www.3gpp.org/ftp/specs/archive/32_series/32.435#measCollec") + doc.appendChild(measCollecFile) + + pi = doc.createProcessingInstruction('xml-stylesheet', + 'type="text/xsl" href="MeasDataCollection.xsl"') + root = doc.firstChild + doc.insertBefore(pi, root) + + fileHeader = doc.createElement('fileHeader') + fileHeader.setAttribute('fileFormatVersion',"32.435 V10.0") + fileHeader.setAttribute('vendorName',"vendor A") + fileHeader.setAttribute('dnPrefix',"SubNetwork=XX") + measCollecFile.appendChild(fileHeader) + + + fileSender = doc.createElement('fileSender') + fileSender.setAttribute('localDn',"test") + fileSender.setAttribute('elementType',"RadioNode") + fileHeader.appendChild(fileSender) + + measCollec = doc.createElement('measCollec') + measCollec.setAttribute('beginTime',time) + fileHeader.appendChild(measCollec) + + measData = doc.createElement('measData') + measCollecFile.appendChild(measData) + + + managedElement = doc.createElement('managedElement') + managedElement.setAttribute('localDn',"test") + managedElement.setAttribute('swVersion',"testversion") + measData.appendChild(managedElement) + + + measInfo = doc.createElement('measInfo') + measInfo.setAttribute('measInfoId',"PM=1,PmGroup=NRCellDU_GNBDU") + measData.appendChild(measInfo) + + + job = doc.createElement('job') + job.setAttribute('jobId',"nr_all") + measInfo.appendChild(job) + + granPeriod = doc.createElement('granPeriod') + granPeriod.setAttribute('duration',"PT900S") + granPeriod.setAttribute('endTime',time) + measInfo.appendChild(granPeriod) + + repPeriod = doc.createElement('repPeriod') + repPeriod.setAttribute('duration',"PT900S") + measInfo.appendChild(repPeriod) + + measurement_index = 1 + for column in row.index: + if column in measurement_list: + measType = doc.createElement('measType') + measType.setAttribute('p',str(measurement_index)) + measTypeValue = doc.createTextNode(column) + measType.appendChild(measTypeValue) + measInfo.appendChild(measType) + + measurement_index = measurement_index + 1 + + measValue = doc.createElement('measValue') + measValue.setAttribute('measObjLdn',"ManagedElement=nodedntest,GNBDUFunction="+ du +",NRCellDU="+ cell) + measInfo.appendChild(measValue) + + measurement_index = 1 + for column in row.index: + if column in measurement_list: + r = doc.createElement('r') + r.setAttribute('p',str(measurement_index)) + measTypeValue = doc.createTextNode(str(row[column])) + r.appendChild(measTypeValue) + measValue.appendChild(r) + + measurement_index = measurement_index + 1 + + xmlstr = doc.toprettyxml(encoding="utf-8") + + format_string = "%Y-%m-%dT%H:%M:%S.%f" + date_string = time + datetimestart = datetime.datetime.strptime(date_string, format_string) + datetimeend = datetimestart + datetime.timedelta(milliseconds=10) + + + datetime_start_formatted = datetimestart.strftime("%Y%m%d.%H%M") + datetime_end_formatted = datetimeend.strftime("%H%M") + secondsfiltered = datetimestart.strftime("%S%f")[:-3] + + filename = dir_name + "/" "A" + datetime_start_formatted + "+0200"+ "-" + datetime_end_formatted + "+0200"+ "_"+ sourcename + "_" + du +"_"+ cell+"_"+ secondsfiltered +".xml" + + with open(filename, "wb") as f: + f.write(xmlstr) + + #Fix to increment timestamp in seconds + datetimestart_in_epoch = int(datetimestart.timestamp()*1000 * 1000) + (index *1000 * 1000) + datetimeend_in_epoch = int(datetimeend.timestamp()*1000 * 1000) + (index *1000 * 1000) + + print(datetimeend_in_epoch) + filename_gz = filename + '.gz' + f_in = open(filename, 'rb') + f_out = gzip.open(filename_gz, 'wb') + f_out.writelines(f_in) + f_out.close() + f_in.close() + return datetimestart_in_epoch,datetimeend_in_epoch,filename_gz + +#Function to copy generated files to http_server +def copy_files_http_server(filename_gz): + copy_command = ["kubectl", "cp", filename_gz ,"ran/pm-https-server-0:/files", "-c", "pm-https-server"] + print(subprocess.run(copy_command, capture_output=True)) + +#Function to push file ready event for each row +def push_file_ready_event(sourcename, filename_gz, datetimestart_in_epoch, datetimeend_in_epoch): + push_file_ready_event_command = ["./push-to-file-ready-topic-qoe.sh",sourcename,filename_gz,str(datetimestart_in_epoch),str(datetimeend_in_epoch)] + print(subprocess.run(push_file_ready_event_command, capture_output=True)) + +#Read input data csv file +if (len(sys.argv) < 4): + sys.exit('Give all input parameters, e.g. python3 qoedatapush.py ') +dir_name='files' +sourcename = sys.argv[1] +max_rows= int(sys.argv[2]) +print('max_rows: ',max_rows) +filtered_cell = sys.argv[3] +print('filtered_cell:',filtered_cell) +if not os.path.exists(dir_name): + os.makedirs(dir_name) + +measurement_list = ['throughput','x','y','availPrbDl','availPrbUl','measPeriodPrb','pdcpBytesUl','pdcpBytesDl','measPeriodPdcpBytes'] +df = pd.read_csv('qoedata.csv') +df_selected = df +df_selected = df_selected[df_selected['nrCellIdentity'] == filtered_cell] +df_selected = df_selected.head(max_rows) +print(df_selected.size) +for index,row in df_selected.iterrows(): + datetimestart_in_epoch,datetimeend_in_epoch,filename_gz = create_xml_document(measurement_list, row, dir_name, sourcename, index) + filename_wo_dir_gz = filename_gz.replace(dir_name+'/','') + copy_files_http_server(filename_gz) + push_file_ready_event(sourcename, filename_wo_dir_gz, datetimestart_in_epoch, datetimeend_in_epoch) -- 2.16.6