From: rajdeep11 Date: Tue, 7 Jan 2025 09:57:44 +0000 (+0530) Subject: Documentation and recipe files X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=commitdiff_plain;h=a7f4fa2c7fb0f75dd319de270a27a146830f62c8;p=aiml-fw%2Faimlfw-dep.git Documentation and recipe files description: 1) Amended the documentation of the AIMLFW 2) Added the recipe files for the K release Change-Id: I5a4801715733f3edd6491f67dbc5bdabcf8627ae Signed-off-by: rajdeep11 --- diff --git a/RECIPE_EXAMPLE/example_recipe_local_images_oran_k_release.yaml b/RECIPE_EXAMPLE/example_recipe_local_images_oran_k_release.yaml new file mode 100644 index 0000000..59c51f8 --- /dev/null +++ b/RECIPE_EXAMPLE/example_recipe_local_images_oran_k_release.yaml @@ -0,0 +1,95 @@ +# ================================================================================== +# +# Copyright (c) 2024 Samsung Electronics Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ================================================================================== + +#------------------------------------------------------------------------- +# Global common setting +#------------------------------------------------------------------------- + +traininghost: + ip_address: +tm: + image: + repository: tm + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + pipelines: {"timeseries": "qoe_pipeline_h_release"} + +leofs: + image: + repository: leofs + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +dataextraction: + image: + repository: data-extraction + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +datalake: + influxdb: + host: + port: + orgname: + bucket: + token: + +kfadapter: + image: + repository: kfadapter + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +aimldashboard: + image: + repository: aiml-dashboard + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + host: + tm_host: "localhost" + notebook_host: "localhost" + debug: "\"false\"" + +aimlnotebook: + image: + repository: aiml-notebook + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +kserveadapter: + image: + repository: kserve-adapter + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "1.0.1" + ric_dms: + ip: + port: 8000 + +modelmgmtservice: + image: + repository: modelmgmtservice + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" diff --git a/RECIPE_EXAMPLE/example_recipe_nexus_images_oran_k_release.yaml b/RECIPE_EXAMPLE/example_recipe_nexus_images_oran_k_release.yaml new file mode 100644 index 0000000..efc22a0 --- /dev/null +++ b/RECIPE_EXAMPLE/example_recipe_nexus_images_oran_k_release.yaml @@ -0,0 +1,95 @@ +# ================================================================================== +# +# Copyright (c) 2024 Samsung Electronics Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ================================================================================== + +#------------------------------------------------------------------------- +# Global common setting +#------------------------------------------------------------------------- + +traininghost: + ip_address: +tm: + image: + repository: nexus3.o-ran-sc.org:10002/o-ran-sc/aiml-fw-awmf-tm-docker + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "3.0.0" + pipelines: {"timeseries": "qoe_pipeline_h_release"} + +leofs: + image: + repository: leofs + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +dataextraction: + image: + repository: nexus3.o-ran-sc.org:10002/o-ran-sc/aiml-fw-athp-data-extraction-docker + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "3.0.0" + +datalake: + influxdb: + host: + port: + orgname: + bucket: + token: + +kfadapter: + image: + repository: nexus3.o-ran-sc.org:10002/o-ran-sc/aiml-fw-athp-tps-kubeflow-adapter-docker + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "3.0.0" + +aimldashboard: + image: + repository: nexus3.o-ran-sc.org:10002/o-ran-sc/portal-aiml-dashboard-docker + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "3.0.0" + host: + tm_host: "localhost" + notebook_host: "localhost" + debug: "\"false\"" + +aimlnotebook: + image: + repository: nexus3.o-ran-sc.org:10002/o-ran-sc/portal-aiml-notebook-docker + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "3.0.0" + +kserveadapter: + image: + repository: nexus3.o-ran-sc.org:10002/o-ran-sc/aiml-fw-aihp-ips-kserve-adapter-docker + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "1.0.1" + ric_dms: + ip: + port: 8000 + +modelmgmtservice: + image: + repository: nexus3.o-ran-sc.org:10002/o-ran-sc/aiml-fw-awmf-modelmgmtservice-docker + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "3.0.0" diff --git a/RECIPE_EXAMPLE/example_recipe_nexus_images_staging_oran_k_release.yaml b/RECIPE_EXAMPLE/example_recipe_nexus_images_staging_oran_k_release.yaml new file mode 100644 index 0000000..1988cbb --- /dev/null +++ b/RECIPE_EXAMPLE/example_recipe_nexus_images_staging_oran_k_release.yaml @@ -0,0 +1,96 @@ +# ================================================================================== +# +# Copyright (c) 2024 Samsung Electronics Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ================================================================================== + +#------------------------------------------------------------------------- +# Global common setting +#------------------------------------------------------------------------- + +traininghost: + ip_address: +tm: + image: + repository: nexus3.o-ran-sc.org:10004/o-ran-sc/aiml-fw-awmf-tm-docker + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "3.0.0" + pipelines: {"timeseries": "qoe_pipeline_h_release"} + +leofs: + image: + repository: leofs + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +dataextraction: + image: + repository: nexus3.o-ran-sc.org:10004/o-ran-sc/aiml-fw-athp-data-extraction-docker + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "3.0.0" + +datalake: + influxdb: + host: + port: + orgname: + bucket: + token: + +kfadapter: + image: + repository: nexus3.o-ran-sc.org:10004/o-ran-sc/aiml-fw-athp-tps-kubeflow-adapter-docker + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "3.0.0" + +aimldashboard: + image: + repository: nexus3.o-ran-sc.org:10004/o-ran-sc/portal-aiml-dashboard-docker + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "3.0.0" + host: + tm_host: "localhost" + notebook_host: "localhost" + debug: "\"false\"" + +aimlnotebook: + image: + repository: nexus3.o-ran-sc.org:10004/o-ran-sc/portal-aiml-notebook-docker + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "3.0.0" + +kserveadapter: + image: + repository: nexus3.o-ran-sc.org:10004/o-ran-sc/aiml-fw-aihp-ips-kserve-adapter-docker + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "1.0.0" + ric_dms: + ip: + port: 8000 + +modelmgmtservice: + image: + repository: nexus3.o-ran-sc.org:10004/o-ran-sc/aiml-fw-awmf-modelmgmtservice-docker + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "3.0.0" + diff --git a/docs/installation-guide.rst b/docs/installation-guide.rst index f899135..da35d65 100755 --- a/docs/installation-guide.rst +++ b/docs/installation-guide.rst @@ -35,6 +35,9 @@ Version history | 2023-12-14 | 1.1.0 | Joseph Thaliath | I release | | | | | | +--------------------+--------------------+--------------------+-----------------------+ +| 2023-12-14 | 2.0.0 | Rajdeep Singh | K release | +| | | | | ++--------------------+--------------------+--------------------+-----------------------+ Introduction @@ -302,252 +305,193 @@ Steps to clear the data in InfluxDB Feature group creation ---------------------- -From AIMLFW dashboard create feature group (Training Jobs-> Create Feature Group ) +From AIMLFW dashboard create feature group (Training Jobs-> Create Feature Group ) Or curl + +NOTE: Here is a curl request to create feature group using curl + +.. code:: bash + + curl --location 'http://:32002/ai-ml-model-training/v1/featureGroup' \ + --header 'Content-Type: application/json' \ + --data '{ + "featuregroup_name": "", + "feature_list": "", + "datalake_source": "InfluxSource", + "enable_dme": , + "host": "", + "port": "",", + "dme_port": "", + "bucket": "", + "token": "", + "source_name": "", + "measured_obj_class": "", + "measurement": "", + "db_org": "" + }' NOTE: Below are some example values to be used for the DME based feature group creation for qoe usecase -+--------------------+-------------------------------------------------------------------+ -| **Parameter** | **Value** | -| | | -+--------------------+-------------------------------------------------------------------+ -| Feature Group Name | featuregroup1 | -| | | -+--------------------+-------------------------------------------------------------------+ -| Features | pdcpBytesDl,pdcpBytesUl | -| | | -+--------------------+-------------------------------------------------------------------+ -| Host | | -| | | -+--------------------+-------------------------------------------------------------------+ -| Port | 31812 | -| | | -+--------------------+-------------------------------------------------------------------+ -| Db Org | est | -| | | -+--------------------+-------------------------------------------------------------------+ -| Bucket Name | pm-logg-bucket | -| | | -+--------------------+-------------------------------------------------------------------+ -| DB Token | | -| | | -+--------------------+-------------------------------------------------------------------+ -| _measurement | test,ManagedElement=nodedntest,GNBDUFunction=1004,NRCellDU=c4_B2 | -| | | -+--------------------+-------------------------------------------------------------------+ -| DME flag | enable it | -| | | -+--------------------+-------------------------------------------------------------------+ -| Source Name | | -| | | -+--------------------+-------------------------------------------------------------------+ -| Measured Obj Class | NRCellDU | -| | | -+--------------------+-------------------------------------------------------------------+ -| Dme port | 31823 | -| | | -+--------------------+-------------------------------------------------------------------+ +.. code:: bash + curl --location ':32002/ai-ml-model-training/v1/featureGroup' \ + --header 'Content-Type: application/json' \ + --data '{ + "featuregroup_name": "", + "feature_list": "x,y,pdcpBytesDl,pdcpBytesUl", + "datalake_source": "InfluxSource", + "enable_dme": true, + "host": "", + "port": "8086", + "dme_port": "31823", + "bucket": "pm-logg-bucket", + "token": "", + "source_name": "", + "measured_obj_class": "NRCellDU", + "measurement": "test,ManagedElement=nodedntest,GNBDUFunction=1004,NRCellDU=c4_B13", + "db_org": "est" + } ' NOTE: Below are some example values to be used for the standalone influx DB creation for qoe usecase. Dme is not used in this example. -+--------------------+-------------------------------------------------------------------+ -| **Parameter** | **Value** | -| | | -+--------------------+-------------------------------------------------------------------+ -| Feature Group Name | featuregroup1 | -| | | -+--------------------+-------------------------------------------------------------------+ -| Features | pdcpBytesDl,pdcpBytesUl | -| | | -+--------------------+-------------------------------------------------------------------+ -| Host | | -| | | -+--------------------+-------------------------------------------------------------------+ -| Port | | -| | | -+--------------------+-------------------------------------------------------------------+ -| Db Org | primary | -| | | -+--------------------+-------------------------------------------------------------------+ -| Bucket Name | UEData | -| | | -+--------------------+-------------------------------------------------------------------+ -| DB Token | | -| | | -+--------------------+-------------------------------------------------------------------+ -| _measurement | liveCell | -| | | -+--------------------+-------------------------------------------------------------------+ - -Register Model (optional) -------------------------- +.. code:: bash + + curl --location 'http://:32002/ai-ml-model-training/v1/featureGroup' \ + --header 'Content-Type: application/json' \ + --data '{ + "featuregroup_name": "", + "feature_list": "pdcpBytesDl,pdcpBytesUl", + "datalake_source": "InfluxSource", + "enable_dme": false, + "host": "my-release-influxdb.default", + "port": "8086", + "dme_port": "", + "bucket": "UEData", + "token": "", + "source_name": "", + "measured_obj_class": "", + "measurement": "liveCell", + "db_org": "primary" + }' + +Register Model (compulsory) +--------------------------- Register the model using the below steps if using Model management service for training. .. code:: bash - curl --location 'http://:32006/registerModel' \ + curl --location 'http://:32006/ai-ml-model-registration/v1/registerModel' \ --header 'Content-Type: application/json' \ --data '{ - "model-name":"qoe1", - "rapp-id": "rapp_1", - "meta-info" : - { - "accuracy":"90", - "model-type":"timeseries", - "feature-list":["pdcpBytesDl","pdcpBytesUl"] - } - }' - -Training job creation with DME as data source ---------------------------------------------- + "modelId": { + "modelName": "modeltest1", + "modelVersion": "1" + }, + "description": "This is a test model.", + "modelInformation": { + "metadata": { + "author": "John Doe" + }, + "inputDataType": "pdcpBytesDl,pdcpBytesUl", + "outputDataType": "pdcpBytesDl,pdcpBytesUl" + } + }' + +Model Discovery +--------------- + +Model discovery can be done using the following API endpoint: + + +To fetch all registered models, use the following API endpoint: + +.. code:: bash + + curl --location 'http://:32006/ai-ml-model-discovery/v1/models' + +To fetch models with model name , use the following API endpoint: + +.. code:: bash + + curl --location 'http://:32006/ai-ml-model-discovery/v1/models?model-name=' + +To fetch specific model, use the following API endpoint: + +.. code:: bash + + curl --location 'http://:32006/ai-ml-model-discovery/v1/models?model-name=&&model-version=' + + +Training job creation with DME or Standalone InfluxDB as data source +-------------------------------------------------------------------- #. AIMLFW should be installed by following steps in section :ref:`Software Installation and Deployment ` #. RANPM setup should be installed and configured as per steps mentioned in section :ref:`Prepare Non-RT RIC DME as data source for AIMLFW ` -#. To create training job, follow the steps in the demo videos stored here: `Training Job creation `__ #. After training job is created and executed successfully, model can be deployed using steps mentioned in section :ref:`Deploy trained qoe prediction model on Kserve ` or :ref:`Steps to deploy model using Kserve adapter ` -NOTE: Below are some example values to be used for the QoE usecase training job creation when model management service is not used. NOTE: The QoE training function does not come pre uploaded, we need to go to training function, create training function and run the qoe-pipeline notebook. -+--------------------+-------------------------------------------------------------------+ -| **Parameter** | **Value** | -| | | -+--------------------+-------------------------------------------------------------------+ -| Training Job Name | qoetest | -| | | -+--------------------+-------------------------------------------------------------------+ -| Model Management | disable | -| Service | | -+--------------------+-------------------------------------------------------------------+ -| Training Function | qoe_pipeline_h_release | -| | | -+--------------------+-------------------------------------------------------------------+ -| FeatureGroup Name | featuregroup1 | -| | | -+--------------------+-------------------------------------------------------------------+ -| Datalake Source | Influx DB | -| | | -+--------------------+-------------------------------------------------------------------+ -| Feature Filter | | -| | | -+--------------------+-------------------------------------------------------------------+ -| Hyper Parameters | epochs:1 | -| | | -+--------------------+-------------------------------------------------------------------+ -| Description | test | -| | | -+--------------------+-------------------------------------------------------------------+ - -NOTE: Below are some example values to be used for the QoE usecase training job creation when model management service is used. - -+--------------------+-------------------------------------------------------------------+ -| **Parameter** | **Value** | -| | | -+--------------------+-------------------------------------------------------------------+ -| Training Job Name | qoetest | -| | | -+--------------------+-------------------------------------------------------------------+ -| Model Management | enable | -| Service | | -+--------------------+-------------------------------------------------------------------+ -| Model name | qoe1 | -| | | -+--------------------+-------------------------------------------------------------------+ -| Datalake Source | Influx DB | -| | | -+--------------------+-------------------------------------------------------------------+ -| Feature Filter | | -| | | -+--------------------+-------------------------------------------------------------------+ -| Hyper Parameters | epochs:1 | -| | | -+--------------------+-------------------------------------------------------------------+ -| Description | test | -| | | -+--------------------+-------------------------------------------------------------------+ - - -Training job creation with standalone Influx DB as data source --------------------------------------------------------------- +.. code:: bash -#. AIMLFW should be installed by following steps in section :ref:`Software Installation and Deployment ` -#. Standalone Influx DB should be setup and configured as mentioned in section :ref:`Install Influx DB as datalake ` -#. To create training job, follow the steps in the demo videos stored here: `Training Job creation `__ -#. After training job is created and executed successfully, model can be deployed using steps mentioned in section :ref:`Deploy trained qoe prediction model on Kserve ` or - :ref:`Steps to deploy model using Kserve adapter ` + curl --location 'http://:32002/ai-ml-model-training/v1/training-jobs' \ + --header 'Content-Type: application/json' \ + --data '{ + "modelId":{ + "modelname": "modeltest15", + "modelversion": "1" + }, + "model_location": "", + "training_config": { + "description": "trainingjob for testing", + "dataPipeline": { + "feature_group_name": "testing_influxdb_01", + "query_filter": "", + "arguments": "{'epochs': 1}" + }, + "trainingPipeline": { + "training_pipeline_name": "qoe_Pipeline_testing_1", + "training_pipeline_version": "qoe_Pipeline_testing_1", + "retraining_pipeline_name":"qoe_Pipeline_retrain", + "retraining_pipeline_version":"2" + } + }, + "training_dataset": "", + "validation_dataset": "", + "notification_url": "", + "consumer_rapp_id": "", + "producer_rapp_id": "" + }' + +.. _reference7: + +Obtain the Status of Training Job +--------------------------------- + +The Status of Trainingjob can be featched using the following API endpoint. Replace with the ID of the training job. -NOTE: Below are some example values to be used for the QoE usecase training job creation when model management service is not used -NOTE: The QoE training function does not come pre uploaded, we need to go to training function, create training function and run the qoe-pipeline notebook. +.. code:: bash + + curl --location http://:32002/ai-ml-model-training/v1/training-jobs//status -+--------------------+-------------------------+ -| **Parameter** | **Value** | -| | | -+--------------------+-------------------------+ -| Training Job Name | qoetest | -| | | -+--------------------+-------------------------+ -| Model Management | disable | -| Service | | -+--------------------+-------------------------+ -| Training Function | qoe_pipeline_g_release | -| | | -+--------------------+-------------------------+ -| FeatureGroup Name | featuregroup1 | -| | | -+--------------------+-------------------------+ -| Datalake Source | Influx DB | -| | | -+--------------------+-------------------------+ -| Feature Filter | | -| | | -+--------------------+-------------------------+ -| Hyper Parameters | epochs:1 | -| | | -+--------------------+-------------------------+ -| Description | test | -| | | -+--------------------+-------------------------+ - -NOTE: Below are some example values to be used for the QoE usecase training job creation when model management service is used - -+--------------------+-------------------------+ -| **Parameter** | **Value** | -| | | -+--------------------+-------------------------+ -| Training Job Name | qoetest | -| | | -+--------------------+-------------------------+ -| Model Management | enable | -| Service | | -+--------------------+-------------------------+ -| Model Name | qoe1 | -| | | -+--------------------+-------------------------+ -| Datalake Source | Influx DB | -| | | -+--------------------+-------------------------+ -| Feature Filter | | -| | | -+--------------------+-------------------------+ -| Hyper Parameters | epochs:1 | -| | | -+--------------------+-------------------------+ -| Description | test | -| | | -+--------------------+-------------------------+ .. _reference5: Obtain Model URL for deploying trained models --------------------------------------------- +URL for deployment can be obainted from AIMFW dashboard (Training Jobs-> Training Job status -> Select Info for a training job -> Model URL) +Or You can curl the following API endpoint to obtain Trainingjob Info and fetch model_url for deployment after training is complete. Replace with the ID of the training job. + .. code:: bash - http://:32002/model////Model.zip + curl --location 'http://:32002/ai-ml-model-training/v1/training-jobs/' + +Or you can directly download the model once the training is complete using the following API endpoint: + +.. code:: bash + curl --location 'http://:32002/model////Model.zip' .. _reference4: