changes for the pipeline

Change-Id: Ia429a4af93ce21546c058171b220fb2131cbf67d
Signed-off-by: rajdeep11 <rajdeep.sin@samsung.com>

diff --git a/kf-pipelines/qoe-pipeline-retrain-2.ipynb b/kf-pipelines/qoe-pipeline-retrain-2.ipynb
index 471eedc..5bfcf92 100644 (file)
--- a/kf-pipelines/qoe-pipeline-retrain-2.ipynb
+++ b/kf-pipelines/qoe-pipeline-retrain-2.ipynb
@@ -29,7 +29,7 @@
    "outputs": [],
    "source": [
     "@component(base_image=BASE_IMAGE,packages_to_install=['requests'])\n",
-    "def train_export_model(featurepath: str, epochs: str, modelname: str, modelversion:str, artifactversion:str):\n",
+    "def train_export_model(featurepath: str, epochs: str, modelname: str, modelversion:str):\n",
     "    \n",
     "    import re\n",
     "    import tensorflow as tf\n",
@@ -85,19 +85,17 @@
     "    print(\"Loading the saved model\")\n",
     "    print(os.listdir(os.getcwd()))\n",
     "    \n",
-    "    pattern = r'(.*?)_(\\d+)$'\n",
-    "\n",
-    "    # Search for the pattern in the input string\n",
-    "    match = re.search(pattern, featurepath)\n",
-    "    trainingjob_id = None\n",
-    "    if match:\n",
-    "        trainingjob_id = int(match.group(2))\n",
-    "        print(\"Training Job ID:\", trainingjob_id)\n",
-    "    else:\n",
-    "        print(\"Pattern not found\")\n",
     "\n",
+    "    url = f\"http://modelmgmtservice.traininghost:8082/models?model-name={modelname}&model-version={modelversion}\"\n",
+    "    modelinfo =  requests.get(url).json()[0]\n",
+    "    artifactversion = modelinfo[\"modelId\"][\"artifactVersion\"]\n",
+    "    model_url = \"\"\n",
+    "    if modelinfo[\"modelLocation\"] != \"\":\n",
+    "        model_url= modelinfo[\"modelLocation\"]\n",
+    "    else :\n",
+    "        model_url = f\"http://tm.traininghost:32002/model/{modelname}/{modelversion}/{artifactversion}/Model.zip\"\n",
     "    # Download the model zip file\n",
-    "    model_url= f\"http://tm.traininghost:32002/model/{trainingjob_id}/Model.zip\"\n",
+    "\n",
     "    print(f\"Downloading model from :{model_url}\")\n",
     "    response = requests.get(model_url)\n",
     "\n",
@@ -180,9 +178,23 @@
     "    data = {}\n",
     "    data['metrics'] = []\n",
     "    data['metrics'].append({'Accuracy': str(np.mean(np.absolute(np.asarray(xx)-np.asarray(yy))<5))})\n",
+    "\n",
+    "# update artifact version\n",
+    "    new_artifactversion =\"\"\n",
+    "    if modelinfo[\"modelLocation\"] != \"\":\n",
+    "        new_artifactversion = \"1.1.0\"\n",
+    "    else:\n",
+    "        major, minor , patch= map(int, artifactversion.split('.'))\n",
+    "        minor+=1\n",
+    "        new_artifactversion = f\"{major}.{minor}.{patch}\"\n",
+    "    \n",
+    "    # update the new artifact version in mme\n",
+    "    url = f\"http://modelmgmtservice.traininghost:8082/model-registrations/updateArtifact/{modelname}/{modelversion}/{new_artifactversion}\"\n",
+    "    updated_model_info= requests.post(url).json()\n",
+    "    print(updated_model_info)\n",
     "    \n",
-    "    mm_sdk.upload_metrics(data, modelname, modelversion,artifactversion)\n",
-    "    mm_sdk.upload_model(\"./\", modelname, modelversion, artifactversion)\n"
+    "    mm_sdk.upload_metrics(data, modelname, modelversion,new_artifactversion)\n",
+    "    mm_sdk.upload_model(\"./\", modelname, modelversion, new_artifactversion)\n"
    ]
   },
   {
@@ -196,9 +208,9 @@
     "    description=\"qoe\",\n",
     ")\n",
     "def super_model_pipeline( \n",
-    "    featurepath: str, epochs: str, modelname: str, modelversion:str, artifactversion:str):\n",
+    "    featurepath: str, epochs: str, modelname: str, modelversion:str):\n",
     "    \n",
-    "    trainop=train_export_model(featurepath=featurepath, epochs=epochs, modelname=modelname, modelversion=modelversion, artifactversion=artifactversion)\n",
+    "    trainop=train_export_model(featurepath=featurepath, epochs=epochs, modelname=modelname, modelversion=modelversion)\n",
     "    trainop.set_caching_options(False)\n",
     "    kubernetes.set_image_pull_policy(trainop, \"IfNotPresent\")"
    ]

diff --git a/kf-pipelines/qoe-pipeline.ipynb b/kf-pipelines/qoe-pipeline.ipynb
index 7d427e3..cac4cc8 100644 (file)
--- a/kf-pipelines/qoe-pipeline.ipynb
+++ b/kf-pipelines/qoe-pipeline.ipynb
@@ -29,7 +29,7 @@
    "outputs": [],
    "source": [
     "@component(base_image=BASE_IMAGE)\n",
-    "def train_export_model(featurepath: str, epochs: str, modelname: str, modelversion:str, modellocation:str):\n",
+    "def train_export_model(featurepath: str, epochs: str, modelname: str, modelversion:str):\n",
     "    \n",
     "    import tensorflow as tf\n",
     "    from numpy import array\n",
@@ -125,9 +125,9 @@
     "    description=\"qoe\",\n",
     ")\n",
     "def super_model_pipeline( \n",
-    "    featurepath: str, epochs: str, modelname: str, modelversion:str, modellocation:str):\n",
+    "    featurepath: str, epochs: str, modelname: str, modelversion:str):\n",
     "    \n",
-    "    trainop=train_export_model(featurepath=featurepath, epochs=epochs, modelname=modelname, modelversion=modelversion, modellocation=modellocation)\n",
+    "    trainop=train_export_model(featurepath=featurepath, epochs=epochs, modelname=modelname, modelversion=modelversion)\n",
     "    trainop.set_caching_options(False)\n",
     "    kubernetes.set_image_pull_policy(trainop, \"IfNotPresent\")"
    ]