Config files for kubeflow Installation.

Issue-Id: AIMLFW-4

Signed-off-by: smahana123 <s.mahana@samsung.com>
Change-Id: Iedd042073fb71bbc658a4abfc1bb3636fdab5dc6

diff --git a/tools/kubeflow/Dockerfile.pipeline b/tools/kubeflow/Dockerfile.pipeline
new file mode 100644 (file)
index 0000000..c9ed8b2
--- /dev/null
+++ b/tools/kubeflow/Dockerfile.pipeline
@@ -0,0 +1,33 @@
+# ==================================================================================
+#
+#       Copyright (c) 2022 Samsung Electronics Co., Ltd. All Rights Reserved.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#          http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# ==================================================================================
+
+FROM python:3.8
+RUN pip3 install protobuf==3.20.1
+RUN pip3 install tensorflow==2.5.0
+RUN pip3 install cassandra-driver==3.25.0
+RUN pip3 install pandas==1.4.4
+RUN pip3 install scikit-learn
+RUN pip3 install boto3
+RUN pip3 install botocore
+RUN pip3 install kubernetes
+ADD SDK /SDK
+WORKDIR /SDK
+RUN pip3 install featurestoresdk_main/.
+RUN pip3 install modelmetricssdk_main/.
+RUN mkdir -p /app_run
+WORKDIR /app_run

diff --git a/tools/kubeflow/config.json b/tools/kubeflow/config.json
new file mode 100644 (file)
index 0000000..244afe5
--- /dev/null
+++ b/tools/kubeflow/config.json
@@ -0,0 +1,22 @@
+{
+  "DBConfig": {
+    "DriverName": "mysql",
+    "DataSourceName": "",
+    "DBName": "mlpipeline",
+    "GroupConcatMaxLen": "4194304"
+  },
+  "ObjectStoreConfig": {
+    "Host": "leofs.kubeflow",
+    "Port": "8080",
+    "AccessKey": "leofs",
+    "SecretAccessKey" : "",
+    "BucketName": "mlpipeline",
+    "PipelinePath": "pipelines"
+  },
+  "ARCHIVE_CONFIG_LOG_FILE_NAME": "main.log",
+  "ARCHIVE_CONFIG_LOG_PATH_PREFIX": "/artifacts",
+  "InitConnectionTimeout": "6m",
+  "DefaultPipelineRunnerServiceAccount": "pipeline-runner",
+  "CacheEnabled": "true",
+  "CRON_SCHEDULE_TIMEZONE": "UTC"
+}

diff --git a/tools/kubeflow/kustomization.yaml b/tools/kubeflow/kustomization.yaml
new file mode 100644 (file)
index 0000000..8aa0d59
--- /dev/null
+++ b/tools/kubeflow/kustomization.yaml
@@ -0,0 +1,21 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+bases:
+  - ../../base
+  - mysql
+
+# Identifier for application manager to apply ownerReference.
+# The ownerReference ensures the resources get garbage collected
+# when application is deleted.
+commonLabels:
+  application-crd-id: kubeflow-pipelines
+
+# !!! If you want to customize the namespace,
+# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml
+namespace: kubeflow
+
+images:
+  - name: mysql
+    newTag: "5.6"
+    newTag: RELEASE.2018-02-09T22-40-05Z

diff --git a/tools/kubeflow/leofs_env.sh b/tools/kubeflow/leofs_env.sh
new file mode 100644 (file)
index 0000000..d68d4f3
--- /dev/null
+++ b/tools/kubeflow/leofs_env.sh
@@ -0,0 +1,19 @@
+# ==================================================================================
+#
+#       Copyright (c) 2022 Samsung Electronics Co., Ltd. All Rights Reserved.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#          http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# ==================================================================================
+export LEOFS_KEY_NAME='leofs'
+export LEOFS_KEY=$(kubectl get secret leofs-secret -n kubeflow -o jsonpath='{.data.password}' | base64 -d)

diff --git a/tools/kubeflow/ml-pipeline-apiserver-deployment.yaml b/tools/kubeflow/ml-pipeline-apiserver-deployment.yaml
new file mode 100644 (file)
index 0000000..38deac9
--- /dev/null
+++ b/tools/kubeflow/ml-pipeline-apiserver-deployment.yaml
@@ -0,0 +1,101 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: ml-pipeline
+  name: ml-pipeline
+spec:
+  selector:
+    matchLabels:
+      app: ml-pipeline
+  template:
+    metadata:
+      labels:
+        app: ml-pipeline
+    spec:
+      containers:
+      - env:
+        - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION
+          valueFrom:
+            configMapKeyRef:
+              name: pipeline-install-config
+              key: autoUpdatePipelineDefaultVersion
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: OBJECTSTORECONFIG_SECURE
+          value: "false"
+        - name: OBJECTSTORECONFIG_BUCKETNAME
+          valueFrom:
+            configMapKeyRef:
+              name: pipeline-install-config
+              key: bucketName
+        - name: DBCONFIG_USER
+          valueFrom:
+            secretKeyRef:
+              name: mysql-secret
+              key: username
+        - name: DBCONFIG_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: mysql-secret
+              key: password
+        - name: DBCONFIG_DBNAME
+          valueFrom:
+            configMapKeyRef:
+              name: pipeline-install-config
+              key: pipelineDb
+        - name: DBCONFIG_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: pipeline-install-config
+              key: dbHost
+        - name: DBCONFIG_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: pipeline-install-config
+              key: dbPort
+        - name: OBJECTSTORECONFIG_ACCESSKEY
+          valueFrom:
+            secretKeyRef:
+              name: mlpipeline-leofs-artifact
+              key: accesskey
+        - name: OBJECTSTORECONFIG_SECRETACCESSKEY
+          valueFrom:
+            secretKeyRef:
+              name: mlpipeline-leofs-artifact
+              key: secretkey
+        image: gcr.io/ml-pipeline/api-server:dummy
+        imagePullPolicy: IfNotPresent
+        name: ml-pipeline-api-server
+        ports:
+        - name: http
+          containerPort: 8888
+        - name: grpc
+          containerPort: 8887
+        readinessProbe:
+          exec:
+            command:
+              - wget
+              - -q # quiet
+              - -S # show server response
+              - -O
+              - "-" # Redirect output to stdout
+              - http://localhost:8888/apis/v1beta1/healthz
+          initialDelaySeconds: 3
+          periodSeconds: 5
+          timeoutSeconds: 2
+        livenessProbe:
+          exec:
+            command:
+              - wget
+              - -q # quiet
+              - -S # show server response
+              - -O
+              - "-" # Redirect output to stdout
+              - http://localhost:8888/apis/v1beta1/healthz
+          initialDelaySeconds: 3
+          periodSeconds: 5
+          timeoutSeconds: 2
+      serviceAccountName: ml-pipeline

diff --git a/tools/kubeflow/mlpipeline-leofs-artifact-secret.yaml b/tools/kubeflow/mlpipeline-leofs-artifact-secret.yaml
new file mode 100644 (file)
index 0000000..9ee4fa5
--- /dev/null
+++ b/tools/kubeflow/mlpipeline-leofs-artifact-secret.yaml
@@ -0,0 +1,24 @@
+# ==================================================================================
+#
+#       Copyright (c) 2022 Samsung Electronics Co., Ltd. All Rights Reserved.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#          http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# ==================================================================================
+kind: Secret
+apiVersion: v1
+metadata:
+  name: mlpipeline-leofs-artifact
+stringData:
+  accesskey: $LEOFS_KEY_NAME
+  secretkey: $LEOFS_KEY

diff --git a/tools/kubeflow/workflow-controller-configmap.yaml b/tools/kubeflow/workflow-controller-configmap.yaml
new file mode 100644 (file)
index 0000000..33c402f
--- /dev/null
+++ b/tools/kubeflow/workflow-controller-configmap.yaml
@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: workflow-controller-configmap
+data:
+  config: |
+    {
+    namespace: $(kfp-namespace),
+    executorImage: gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance,
+    containerRuntimeExecutor: $(kfp-container-runtime-executor),
+    artifactRepository:
+    {
+        s3: {
+            bucket: $(kfp-artifact-bucket-name),
+            keyPrefix: artifacts,
+            endpoint: leofs.$(kfp-namespace):8080,
+            insecure: true,
+            accessKeySecret: {
+                name: mlpipeline-leofs-artifact,
+                key: accesskey
+            },
+            secretKeySecret: {
+                name: mlpipeline-leofs-artifact,
+                key: secretkey
+            }
+        },
+        archiveLogs: true
+    }
+    }