qoe_pipeline_retrain fails with TF 2.20.0

qoe_pipeline and qoe_pipeline_retrain pipelines we will be saving the model in keras and SavedModel formats. Retraining pipeline uses keras format and deployment uses SavedModel format. Testing scenarios:https://lf-o-ran-sc.atlassian.net/browse/AIMLFW-246?focusedCommentId=15178

Issue-ID: AIMLFW-246

Change-Id: I8f86ada474f1fe5a60d6daef19ab55c69b0acfbf
Signed-off-by: Swaraj Kumar <swaraj.kumar@samsung.com>

diff --git a/kf-pipelines/qoe-pipeline-retrain-2.ipynb b/kf-pipelines/qoe-pipeline-retrain.ipynb
similarity index 89%
rename from kf-pipelines/qoe-pipeline-retrain-2.ipynb
rename to kf-pipelines/qoe-pipeline-retrain.ipynb
index e339505..f9a2d3a 100644 (file)
--- a/kf-pipelines/qoe-pipeline-retrain-2.ipynb
+++ b/kf-pipelines/qoe-pipeline-retrain.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -15,7 +15,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -93,7 +93,8 @@
     "    if modelinfo[\"modelLocation\"] != \"\":\n",
     "        model_url= modelinfo[\"modelLocation\"]\n",
     "    else :\n",
-    "        model_url = f\"http://tm.traininghost:32002/model/{modelname}/{modelversion}/{artifactversion}/Model.zip\"\n",
+    "        keras_model= modelname + \"_keras\"\n",
+    "        model_url = f\"http://tm.traininghost:32002/model/{keras_model}/{modelversion}/{artifactversion}/Model.zip\"\n",
     "    # Download the model zip file\n",
     "\n",
     "    print(f\"Downloading model from :{model_url}\")\n",
@@ -130,7 +131,7 @@
     "        print(f'Zip file not found: {zip_file_path}')\n",
     "\n",
     "    # Path to the directory containing the saved model\n",
-    "    model_path = f\"./Model/{modelversion}\"\n",
+    "    model_path = f\"./Model/{modelversion}/model.keras\"\n",
     "\n",
     "    # Load the model in SavedModel format     \n",
     "    model = tf.keras.models.load_model(model_path)\n",
@@ -169,13 +170,16 @@
     "    xx = y\n",
     "    yy = yhat\n",
     "    \n",
-    "    retrained_model_path = \"./retrain\"\n",
-    "    if not os.path.exists(retrained_model_path):\n",
-    "        os.makedirs(retrained_model_path)\n",
-    "\n",
-    "    # Save the retrained model\n",
-    "    model.save(retrained_model_path)\n",
-    "    print(f\"Retrained model saved at {retrained_model_path}\")\n",
+    "    print(\"Saving models ...\")\n",
+    "    save_directory = './retrain/keras_model'\n",
+    "    if not os.path.exists(save_directory):\n",
+    "        os.makedirs(save_directory, exist_ok=True)\n",
+    "        print(f\"Created directory: {save_directory}\")\n",
+    "    else:\n",
+    "        print(f\"Directory already exists: {save_directory}\")\n",
+    "        \n",
+    "    model.save('./retrain/keras_model/model.keras')\n",
+    "    model.export('./retrain/saved_model')\n",
     "\n",
     "    import json\n",
     "    data = {}\n",
@@ -196,8 +200,11 @@
     "    updated_model_info= requests.post(url).json()\n",
     "    print(updated_model_info)\n",
     "    \n",
-    "    mm_sdk.upload_metrics(data, modelname, modelversion,new_artifactversion)\n",
-    "    mm_sdk.upload_model(\"./retrain/\", modelname, modelversion, new_artifactversion)\n"
+    "    print(\"uploading keras model to MME\")\n",
+    "    mm_sdk.upload_model(\"./retrain/keras_model\", modelname + \"_keras\", modelversion, new_artifactversion)\n",
+    "    print(\"Saved keras format\")\n",
+    "    mm_sdk.upload_model(\"./retrain/saved_model\", modelname, modelversion, new_artifactversion)\n",
+    "    print(\"Saved savedmodel format\")"
    ]
   },
   {

diff --git a/kf-pipelines/qoe-pipeline.ipynb b/kf-pipelines/qoe-pipeline.ipynb
index ea907a6..36af090 100644 (file)
--- a/kf-pipelines/qoe-pipeline.ipynb
+++ b/kf-pipelines/qoe-pipeline.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -15,7 +15,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -24,7 +24,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -98,15 +98,24 @@
     "    \n",
     "    xx = y\n",
     "    yy = yhat\n",
-    "    model_save_filepath = \"./\"\n",
-    "    model.export(model_save_filepath)\n",
+    "    \n",
+    "    print(\"Saving models ...\")\n",
+    "    save_directory = './keras_model'\n",
+    "    if not os.path.exists(save_directory):\n",
+    "        os.makedirs(save_directory, exist_ok=True)\n",
+    "        print(f\"Created directory: {save_directory}\")\n",
+    "    else:\n",
+    "        print(f\"Directory already exists: {save_directory}\")\n",
+    "        \n",
+    "    model.save('./keras_model/model.keras')\n",
+    "    model.export('./saved_model')\n",
     "    \n",
     "    import json\n",
     "    data = {}\n",
     "    data['metrics'] = []\n",
     "    data['metrics'].append({'Accuracy': str(np.mean(np.absolute(np.asarray(xx)-np.asarray(yy))<5))})\n",
     "    \n",
-    "    #as new artifact after training will always be 1.0.0\n",
+    "#     as new artifact after training will always be 1.0.0\n",
     "    artifactversion=\"1.0.0\"\n",
     "    url = f\"http://modelmgmtservice.traininghost:8082/ai-ml-model-registration/v1/model-registrations/updateArtifact/{modelname}/{modelversion}/{artifactversion}\"\n",
     "    updated_model_info= requests.post(url).json()\n",
@@ -116,13 +125,16 @@
     "    trainingjob_id = featurepath.split('_')[-1]\n",
     "    mm_sdk.upload_metrics(data, trainingjob_id)\n",
     "    print(\"Model-metric : \", mm_sdk.get_metrics(trainingjob_id))\n",
-    "    mm_sdk.upload_model(model_save_filepath, modelname, modelversion, artifactversion)\n",
-    "    "
+    "    print(\"uploading keras model to MME\")\n",
+    "    mm_sdk.upload_model(\"./keras_model\", modelname + \"_keras\", modelversion, artifactversion)\n",
+    "    print(\"Saved keras format\")\n",
+    "    mm_sdk.upload_model(\"./saved_model\", modelname, modelversion, artifactversion)\n",
+    "    print(\"Saved savedmodel format\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -140,7 +152,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -153,20 +165,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "<Response [200]>"
-      ]
-     },
-     "execution_count": 24,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "import requests\n",
     "pipeline_name=\"qoe_Pipeline\"\n",