Add practice of building o2ims images inside a container 85/7085/8
authorBin Yang <bin.yang@windriver.com>
Fri, 19 Nov 2021 02:56:09 +0000 (10:56 +0800)
committerBin Yang <bin.yang@windriver.com>
Sun, 21 Nov 2021 03:52:33 +0000 (11:52 +0800)
Add helm chart for k8s deployment

Signed-off-by: Bin Yang <bin.yang@windriver.com>
Change-Id: Iae4501c7a489be9266771d304f6a787c5992169b

16 files changed:
README-o2imsbuilder.md [new file with mode: 0644]
charts/.helmignore [new file with mode: 0644]
charts/Chart.yaml [new file with mode: 0644]
charts/resources/scripts/init/o2api_start.sh [new file with mode: 0644]
charts/resources/scripts/init/o2pubsub_start.sh [new file with mode: 0644]
charts/resources/scripts/init/o2watcher_start.sh [new file with mode: 0644]
charts/resources/scripts/init/postgres_start.sh [new file with mode: 0644]
charts/templates/.helmignore [new file with mode: 0644]
charts/templates/_helpers.tpl [new file with mode: 0644]
charts/templates/configmap.yaml [new file with mode: 0644]
charts/templates/deployment.yaml [new file with mode: 0644]
charts/templates/service.yaml [new file with mode: 0644]
charts/values.yaml [new file with mode: 0644]
docker-compose.yml
o2ims/bootstrap.py
requirements.txt

diff --git a/README-o2imsbuilder.md b/README-o2imsbuilder.md
new file mode 100644 (file)
index 0000000..e19b759
--- /dev/null
@@ -0,0 +1,60 @@
+\r
+## build o2ims from a container over INF\r
+\r
+\r
+## bring up container\r
+\r
+## Important: make sure container and host shares the same filepath to overcome local dir mounting issue\r
+\r
+mkdir -p /home/sysadmin/share\r
+sudo docker run -dt --privileged -v /home/sysadmin/share/:/home/sysadmin/share/ -v /var/run:/var/run --name o2imsbuilder2 centos:7\r
+\r
+## build inside container\r
+sudo docker exec -it o2imsbuilder2 bash\r
+\r
+curl -L https://get.daocloud.io/docker/compose/releases/download/1.25.4/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose\r
+chmod +x /usr/local/bin/docker-compose\r
+docker-compose -v\r
+\r
+yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo\r
+yum makecache fast\r
+yum install -y docker-ce\r
+docker ps\r
+\r
+yum install -y git\r
+\r
+cd /home/sysadmin/share/\r
+git clone "https://gerrit.o-ran-sc.org/r/pti/o2"\r
+cd o2\r
+\r
+mkdir -p temp\r
+cd temp\r
+git clone https://opendev.org/starlingx/config.git\r
+git clone https://opendev.org/starlingx/distcloud-client.git\r
+cd -\r
+\r
+docker-compose build\r
+\r
+## test over inf host\r
+export NAMESPACE=orano2\r
+kubectl create ns ${NAMESPACE}\r
+\r
+source /etc/platform/openrc\r
+sudo docker login registry.local:9001 -u ${OS_PROJECT_NAME} -p ${OS_PASSWORD}\r
+\r
+\r
+kubectl -n ${NAMESPACE} create secret docker-registry ${OS_PROJECT_NAME}-${NAMESPACE}-registry-secret \\r
+--docker-server=registry.local:9001 --docker-username=${OS_PROJECT_NAME} \\r
+--docker-password=${OS_PASSWORD} --docker-email=noreply@windriver.com\r
+\r
+==> secret/admin-orano2-registry-secret created\r
+\r
+sudo docker tag o2imsdms:latest registry.local:9001/admin/o2imsdms:0.1.1\r
+sudo docker image push registry.local:9001/admin/o2imsdms:0.1.1\r
+\r
+cd /home/sysadmin/share/o2\r
+helm install o2imstest charts\r
+kubectl -n ${NAMESPACE} get pods\r
+\r
+\r
+## issues:\r
diff --git a/charts/.helmignore b/charts/.helmignore
new file mode 100644 (file)
index 0000000..50af031
--- /dev/null
@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/charts/Chart.yaml b/charts/Chart.yaml
new file mode 100644 (file)
index 0000000..31f4f32
--- /dev/null
@@ -0,0 +1,19 @@
+# Copyright (C) 2021 Wind River Systems, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart to deploy O2 Services
+name: orano2
+version: 0.1.0
diff --git a/charts/resources/scripts/init/o2api_start.sh b/charts/resources/scripts/init/o2api_start.sh
new file mode 100644 (file)
index 0000000..46ea5f5
--- /dev/null
@@ -0,0 +1,34 @@
+# Copyright (C) 2021 Wind River Systems, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+#!/bin/bash
+
+# pull latest code to debug
+cd /root/
+git clone "https://gerrit.o-ran-sc.org/r/pti/o2"
+# cd o2
+# git pull https://gerrit.o-ran-sc.org/r/pti/o2 refs/changes/85/7085/5
+# pip install retry
+
+pip install -e /root/o2
+
+cat <<EOF>>/etc/hosts
+127.0.0.1  api
+127.0.0.1  postgres
+127.0.0.1  redis
+EOF
+
+flask run --host=0.0.0.0 --port=80
+
+sleep infinity
diff --git a/charts/resources/scripts/init/o2pubsub_start.sh b/charts/resources/scripts/init/o2pubsub_start.sh
new file mode 100644 (file)
index 0000000..e39329b
--- /dev/null
@@ -0,0 +1,24 @@
+# Copyright (C) 2021 Wind River Systems, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+#!/bin/bash
+
+# pull latest code to debug
+cd /root/
+git clone "https://gerrit.o-ran-sc.org/r/pti/o2"
+pip install -e /root/o2
+
+python /root/o2/o2ims/entrypoints/redis_eventconsumer.py
+
+sleep infinity
diff --git a/charts/resources/scripts/init/o2watcher_start.sh b/charts/resources/scripts/init/o2watcher_start.sh
new file mode 100644 (file)
index 0000000..53fd670
--- /dev/null
@@ -0,0 +1,24 @@
+# Copyright (C) 2021 Wind River Systems, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+#!/bin/bash
+
+# pull latest code to debug
+cd /root/
+git clone "https://gerrit.o-ran-sc.org/r/pti/o2"
+pip install -e /root/o2
+
+python /root/o2/o2ims/entrypoints/resource_watcher.py
+
+sleep infinity
diff --git a/charts/resources/scripts/init/postgres_start.sh b/charts/resources/scripts/init/postgres_start.sh
new file mode 100644 (file)
index 0000000..975c0ac
--- /dev/null
@@ -0,0 +1,24 @@
+# Copyright (C) 2021 Wind River Systems, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+#!/bin/bash
+
+# sed -i 's/huge_page=try/huge_page=off/' /usr/share/postgresql/postgresql.conf.sample
+cat <<EOF >> /usr/share/postgresql/postgresql.conf.sample
+huge_pages = off
+EOF
+
+/docker-entrypoint.sh postgres
+
+sleep infinity
diff --git a/charts/templates/.helmignore b/charts/templates/.helmignore
new file mode 100644 (file)
index 0000000..50af031
--- /dev/null
@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/charts/templates/_helpers.tpl b/charts/templates/_helpers.tpl
new file mode 100644 (file)
index 0000000..596f567
--- /dev/null
@@ -0,0 +1,49 @@
+{{/*
+# Copyright (C) 2021 Wind River Systems, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+*/}}
+
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "orano2.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "orano2.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "orano2.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
diff --git a/charts/templates/configmap.yaml b/charts/templates/configmap.yaml
new file mode 100644 (file)
index 0000000..8ff60d8
--- /dev/null
@@ -0,0 +1,26 @@
+# Copyright (C) 2021 Wind River Systems, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Chart.Name }}-scripts-configmap
+  namespace: {{ .Values.global.namespace }}
+  labels:
+    release: {{ .Release.Name }}
+    app: {{ include "orano2.name" . }}
+    chart: {{ .Chart.Name }}
+data:
+{{ tpl (.Files.Glob "resources/scripts/init/*").AsConfig . | indent 2 }}
diff --git a/charts/templates/deployment.yaml b/charts/templates/deployment.yaml
new file mode 100644 (file)
index 0000000..2e8ebf1
--- /dev/null
@@ -0,0 +1,131 @@
+# Copyright (C) 2021 Wind River Systems, Inc.\r
+#\r
+#  Licensed under the Apache License, Version 2.0 (the "License");\r
+#  you may not use this file except in compliance with the License.\r
+#  You may obtain a copy of the License at\r
+#\r
+#      http://www.apache.org/licenses/LICENSE-2.0\r
+#\r
+#  Unless required by applicable law or agreed to in writing, software\r
+#  distributed under the License is distributed on an "AS IS" BASIS,\r
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r
+#  See the License for the specific language governing permissions and\r
+#  limitations under the License.\r
+\r
+---\r
+apiVersion: apps/v1\r
+kind: Deployment\r
+metadata:\r
+  name: o2api\r
+  namespace: {{ .Values.global.namespace }}\r
+  labels:\r
+    app: o2api\r
+spec:\r
+  replicas: 1\r
+  selector:\r
+    matchLabels:\r
+      app: o2api\r
+  template:\r
+    metadata:\r
+      labels:\r
+        app: o2api\r
+    spec:\r
+      imagePullSecrets:\r
+        - name: {{ .Values.o2ims.imagePullSecrets }}\r
+{{- if .Values.o2ims.affinity }}\r
+      affinity:\r
+{{ toYaml .Values.o2ims.affinity | indent 8 }}\r
+{{- end }}\r
+      containers:\r
+        - name: postgres\r
+          image: postgres:9.6\r
+          ports:\r
+            - containerPort: 5432\r
+          env:\r
+            - name: POSTGRES_PASSWORD\r
+              value: o2ims123\r
+            - name: POSTGRES_USER\r
+              value: o2ims\r
+          command: ["/bin/bash", "/opt/postgres_start.sh"]\r
+          volumeMounts:\r
+            - name: scripts\r
+              mountPath: /opt\r
+        - name: o2api\r
+          image: "{{ .Values.o2ims.image.repository }}:{{ .Values.o2ims.image.tag }}"\r
+          ports:\r
+            - containerPort: 80\r
+          env:\r
+            - name: API_HOST\r
+              value: api\r
+            - name: DB_HOST\r
+              value: postgres\r
+            - name: DB_PASSWORD\r
+              value: o2ims123\r
+            - name: FLASK_APP\r
+              value: /root/o2/o2ims/entrypoints/flask_application.py\r
+            - name: FLASK_DEBUG\r
+              value: "1"\r
+            - name: LOGGING_CONFIG_LEVEL\r
+              value: DEBUG\r
+            - name: OS_AUTH_URL\r
+            - name: OS_PASSWORD\r
+            - name: OS_USERNAME\r
+            - name: PYTHONDONTWRITEBYTECODE\r
+              value: "1"\r
+            - name: PYTHONUNBUFFERED\r
+              value: "1"\r
+            - name: REDIS_HOST\r
+              value: redis\r
+          command: ["/bin/bash", "/opt/o2api_start.sh"]\r
+          volumeMounts:\r
+            - name: scripts\r
+              mountPath: /opt\r
+        - name: redis\r
+          image: redis:alpine\r
+          ports:\r
+            - containerPort: 6379\r
+        - name: watcher\r
+          image: "{{ .Values.o2ims.image.repository }}:{{ .Values.o2ims.image.tag }}"\r
+          command: ["/bin/bash", "/opt/o2watcher_start.sh"]\r
+          env:\r
+            - name: DB_HOST\r
+              value: postgres\r
+            - name: DB_PASSWORD\r
+              value: o2ims123\r
+            - name: LOGGING_CONFIG_LEVEL\r
+              value: DEBUG\r
+            - name: OS_AUTH_URL\r
+            - name: OS_PASSWORD\r
+            - name: OS_USERNAME\r
+            - name: PYTHONDONTWRITEBYTECODE\r
+              value: "1"\r
+            - name: REDIS_HOST\r
+              value: redis\r
+          volumeMounts:\r
+            - name: scripts\r
+              mountPath: /opt\r
+        - name: o2pubsub\r
+          image: "{{ .Values.o2ims.image.repository }}:{{ .Values.o2ims.image.tag }}"\r
+          command: ["/bin/bash", "/opt/o2pubsub_start.sh"]\r
+          env:\r
+            - name: DB_HOST\r
+              value: postgres\r
+            - name: DB_PASSWORD\r
+              value: o2ims123\r
+            - name: LOGGING_CONFIG_LEVEL\r
+              value: DEBUG\r
+            - name: OS_AUTH_URL\r
+            - name: OS_PASSWORD\r
+            - name: OS_USERNAME\r
+            - name: PYTHONDONTWRITEBYTECODE\r
+              value: "1"\r
+            - name: REDIS_HOST\r
+              value: redis\r
+          volumeMounts:\r
+            - name: scripts\r
+              mountPath: /opt\r
+      volumes:\r
+        - name: scripts\r
+          configMap:\r
+            name: {{ .Chart.Name }}-scripts-configmap\r
+---\r
diff --git a/charts/templates/service.yaml b/charts/templates/service.yaml
new file mode 100644 (file)
index 0000000..7352918
--- /dev/null
@@ -0,0 +1,30 @@
+# Copyright (C) 2021 Wind River Systems, Inc.\r
+#\r
+#  Licensed under the Apache License, Version 2.0 (the "License");\r
+#  you may not use this file except in compliance with the License.\r
+#  You may obtain a copy of the License at\r
+#\r
+#      http://www.apache.org/licenses/LICENSE-2.0\r
+#\r
+#  Unless required by applicable law or agreed to in writing, software\r
+#  distributed under the License is distributed on an "AS IS" BASIS,\r
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r
+#  See the License for the specific language governing permissions and\r
+#  limitations under the License.\r
+#\r
+---\r
+apiVersion: v1\r
+kind: Service\r
+metadata:\r
+  name: o2api\r
+  namespace: {{ .Values.global.namespace }}\r
+spec:\r
+  #clusterIP: None\r
+  ports:\r
+  - name: o2api\r
+    port: 5005\r
+    targetPort: 5005\r
+    protocol: TCP\r
+  selector:\r
+    app: o2api\r
+---\r
diff --git a/charts/values.yaml b/charts/values.yaml
new file mode 100644 (file)
index 0000000..2772908
--- /dev/null
@@ -0,0 +1,37 @@
+# Copyright (C) 2021 Wind River Systems, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+# Default values for O2 services.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+
+nameOverride: ""
+fullnameOverride: ""
+
+resources:
+  cpu: 1
+  memory: 2Gi
+
+global:
+  namespace: orano2
+
+o2ims:
+  imagePullSecrets: admin-orano2-registry-secret
+  image:
+    repository: registry.local:9001/admin/o2imsdms
+    tag: 0.1.1
+    pullPolicy: IfNotPresent
index 02190b2..64334b7 100644 (file)
@@ -6,7 +6,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.localtest
-    image: o2imsdms-image
+    image: o2imsdms
     depends_on:
       - postgres
       - redis
@@ -30,7 +30,7 @@ services:
       - /tests/o2ims-redis-entry.sh
 
   api:
-    image: o2imsdms-image
+    image: o2imsdms
     depends_on:
       - redis_pubsub
     environment:
@@ -64,10 +64,9 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.localtest
-    image: o2imsdms-image
+    image: o2imsdms
     depends_on:
-      - postgres
-      - redis
+      - redis_pubsub
     environment:
       - DB_HOST=postgres
       - DB_PASSWORD=o2ims123
index 10bdc26..595fd91 100644 (file)
@@ -12,6 +12,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+from retry import retry
 import inspect
 from typing import Callable
 from o2ims.adapter import orm, redis_eventpublisher
@@ -21,6 +22,16 @@ from o2ims.adapter.notifications import AbstractNotifications,\
 from o2ims.service import handlers, messagebus, unit_of_work
 from o2ims.adapter.unit_of_work import SqlAlchemyUnitOfWork
 from o2ims.adapter.clients import orm_stx
+from o2common.helper import o2logging
+logger = o2logging.get_logger(__name__)
+
+
+@retry(tries=100, delay=2, backoff=1)
+def wait_for_db_ready(engine):
+    # wait for db up
+    logger.info("Wait for DB ready ...")
+    engine.connect()
+    logger.info("DB is ready")
 
 
 def bootstrap(
@@ -34,12 +45,14 @@ def bootstrap(
         notifications = SmoO2Notifications()
 
     if start_orm:
-        orm_stx.start_o2ims_stx_mappers(uow)
         with uow:
             # get default engine if uow is by default
             engine = uow.session.get_bind()
+            wait_for_db_ready(engine)
             orm.start_o2ims_mappers(engine)
 
+        orm_stx.start_o2ims_stx_mappers(uow)
+
     dependencies = {"uow": uow, "notifications": notifications,
                     "publish": publish}
     injected_event_handlers = {
index e173bd4..fae9d70 100644 (file)
@@ -12,5 +12,5 @@ Cython>=3.0a1
 httplib2\r
 babel\r
 PrettyTable<0.8,>=0.7.2\r
-# -e git+https://opendev.org/starlingx/distcloud-client.git@master#egg=distributedcloud-client&subdirectory=distributedcloud-client\r
-# -e git+https://opendev.org/starlingx/config.git@master#egg=cgtsclient&subdirectory=sysinv/cgts-client/cgts-client#\r
+\r
+retry\r