From 7237732678da624e0807b2010afb9e16dea730be Mon Sep 17 00:00:00 2001 From: smahana123 Date: Thu, 17 Nov 2022 16:18:23 +0530 Subject: [PATCH] Config files for kubeflow Installation. Issue-Id: AIMLFW-4 Signed-off-by: smahana123 Change-Id: Iedd042073fb71bbc658a4abfc1bb3636fdab5dc6 --- tools/kubeflow/Dockerfile.pipeline | 33 +++++++ tools/kubeflow/config.json | 22 +++++ tools/kubeflow/kustomization.yaml | 21 +++++ tools/kubeflow/leofs_env.sh | 19 ++++ .../kubeflow/ml-pipeline-apiserver-deployment.yaml | 101 +++++++++++++++++++++ .../kubeflow/mlpipeline-leofs-artifact-secret.yaml | 24 +++++ tools/kubeflow/workflow-controller-configmap.yaml | 29 ++++++ 7 files changed, 249 insertions(+) create mode 100644 tools/kubeflow/Dockerfile.pipeline create mode 100644 tools/kubeflow/config.json create mode 100644 tools/kubeflow/kustomization.yaml create mode 100644 tools/kubeflow/leofs_env.sh create mode 100644 tools/kubeflow/ml-pipeline-apiserver-deployment.yaml create mode 100644 tools/kubeflow/mlpipeline-leofs-artifact-secret.yaml create mode 100644 tools/kubeflow/workflow-controller-configmap.yaml diff --git a/tools/kubeflow/Dockerfile.pipeline b/tools/kubeflow/Dockerfile.pipeline new file mode 100644 index 0000000..c9ed8b2 --- /dev/null +++ b/tools/kubeflow/Dockerfile.pipeline @@ -0,0 +1,33 @@ +# ================================================================================== +# +# Copyright (c) 2022 Samsung Electronics Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ================================================================================== + +FROM python:3.8 +RUN pip3 install protobuf==3.20.1 +RUN pip3 install tensorflow==2.5.0 +RUN pip3 install cassandra-driver==3.25.0 +RUN pip3 install pandas==1.4.4 +RUN pip3 install scikit-learn +RUN pip3 install boto3 +RUN pip3 install botocore +RUN pip3 install kubernetes +ADD SDK /SDK +WORKDIR /SDK +RUN pip3 install featurestoresdk_main/. +RUN pip3 install modelmetricssdk_main/. +RUN mkdir -p /app_run +WORKDIR /app_run diff --git a/tools/kubeflow/config.json b/tools/kubeflow/config.json new file mode 100644 index 0000000..244afe5 --- /dev/null +++ b/tools/kubeflow/config.json @@ -0,0 +1,22 @@ +{ + "DBConfig": { + "DriverName": "mysql", + "DataSourceName": "", + "DBName": "mlpipeline", + "GroupConcatMaxLen": "4194304" + }, + "ObjectStoreConfig": { + "Host": "leofs.kubeflow", + "Port": "8080", + "AccessKey": "leofs", + "SecretAccessKey" : "", + "BucketName": "mlpipeline", + "PipelinePath": "pipelines" + }, + "ARCHIVE_CONFIG_LOG_FILE_NAME": "main.log", + "ARCHIVE_CONFIG_LOG_PATH_PREFIX": "/artifacts", + "InitConnectionTimeout": "6m", + "DefaultPipelineRunnerServiceAccount": "pipeline-runner", + "CacheEnabled": "true", + "CRON_SCHEDULE_TIMEZONE": "UTC" +} diff --git a/tools/kubeflow/kustomization.yaml b/tools/kubeflow/kustomization.yaml new file mode 100644 index 0000000..8aa0d59 --- /dev/null +++ b/tools/kubeflow/kustomization.yaml @@ -0,0 +1,21 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +bases: + - ../../base + - mysql + +# Identifier for application manager to apply ownerReference. +# The ownerReference ensures the resources get garbage collected +# when application is deleted. +commonLabels: + application-crd-id: kubeflow-pipelines + +# !!! If you want to customize the namespace, +# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml +namespace: kubeflow + +images: + - name: mysql + newTag: "5.6" + newTag: RELEASE.2018-02-09T22-40-05Z diff --git a/tools/kubeflow/leofs_env.sh b/tools/kubeflow/leofs_env.sh new file mode 100644 index 0000000..d68d4f3 --- /dev/null +++ b/tools/kubeflow/leofs_env.sh @@ -0,0 +1,19 @@ +# ================================================================================== +# +# Copyright (c) 2022 Samsung Electronics Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ================================================================================== +export LEOFS_KEY_NAME='leofs' +export LEOFS_KEY=$(kubectl get secret leofs-secret -n kubeflow -o jsonpath='{.data.password}' | base64 -d) diff --git a/tools/kubeflow/ml-pipeline-apiserver-deployment.yaml b/tools/kubeflow/ml-pipeline-apiserver-deployment.yaml new file mode 100644 index 0000000..38deac9 --- /dev/null +++ b/tools/kubeflow/ml-pipeline-apiserver-deployment.yaml @@ -0,0 +1,101 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: ml-pipeline + name: ml-pipeline +spec: + selector: + matchLabels: + app: ml-pipeline + template: + metadata: + labels: + app: ml-pipeline + spec: + containers: + - env: + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: autoUpdatePipelineDefaultVersion + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OBJECTSTORECONFIG_SECURE + value: "false" + - name: OBJECTSTORECONFIG_BUCKETNAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: bucketName + - name: DBCONFIG_USER + valueFrom: + secretKeyRef: + name: mysql-secret + key: username + - name: DBCONFIG_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-secret + key: password + - name: DBCONFIG_DBNAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: pipelineDb + - name: DBCONFIG_HOST + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: dbHost + - name: DBCONFIG_PORT + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: dbPort + - name: OBJECTSTORECONFIG_ACCESSKEY + valueFrom: + secretKeyRef: + name: mlpipeline-leofs-artifact + key: accesskey + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + valueFrom: + secretKeyRef: + name: mlpipeline-leofs-artifact + key: secretkey + image: gcr.io/ml-pipeline/api-server:dummy + imagePullPolicy: IfNotPresent + name: ml-pipeline-api-server + ports: + - name: http + containerPort: 8888 + - name: grpc + containerPort: 8887 + readinessProbe: + exec: + command: + - wget + - -q # quiet + - -S # show server response + - -O + - "-" # Redirect output to stdout + - http://localhost:8888/apis/v1beta1/healthz + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + livenessProbe: + exec: + command: + - wget + - -q # quiet + - -S # show server response + - -O + - "-" # Redirect output to stdout + - http://localhost:8888/apis/v1beta1/healthz + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + serviceAccountName: ml-pipeline diff --git a/tools/kubeflow/mlpipeline-leofs-artifact-secret.yaml b/tools/kubeflow/mlpipeline-leofs-artifact-secret.yaml new file mode 100644 index 0000000..9ee4fa5 --- /dev/null +++ b/tools/kubeflow/mlpipeline-leofs-artifact-secret.yaml @@ -0,0 +1,24 @@ +# ================================================================================== +# +# Copyright (c) 2022 Samsung Electronics Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ================================================================================== +kind: Secret +apiVersion: v1 +metadata: + name: mlpipeline-leofs-artifact +stringData: + accesskey: $LEOFS_KEY_NAME + secretkey: $LEOFS_KEY diff --git a/tools/kubeflow/workflow-controller-configmap.yaml b/tools/kubeflow/workflow-controller-configmap.yaml new file mode 100644 index 0000000..33c402f --- /dev/null +++ b/tools/kubeflow/workflow-controller-configmap.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: workflow-controller-configmap +data: + config: | + { + namespace: $(kfp-namespace), + executorImage: gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance, + containerRuntimeExecutor: $(kfp-container-runtime-executor), + artifactRepository: + { + s3: { + bucket: $(kfp-artifact-bucket-name), + keyPrefix: artifacts, + endpoint: leofs.$(kfp-namespace):8080, + insecure: true, + accessKeySecret: { + name: mlpipeline-leofs-artifact, + key: accesskey + }, + secretKeySecret: { + name: mlpipeline-leofs-artifact, + key: secretkey + } + }, + archiveLogs: true + } + } -- 2.16.6