From: Jackie Huang Date: Mon, 9 Dec 2024 05:27:52 +0000 (+0800) Subject: patches-arm: add patches based on stx.9.0 for ansible-playbooks X-Git-Tag: k-release~12 X-Git-Url: https://gerrit.o-ran-sc.org/r/gitweb?a=commitdiff_plain;h=aead9f3dce362e99a1d99a3398c3827b0cf654ea;p=pti%2Frtp.git patches-arm: add patches based on stx.9.0 for ansible-playbooks Add patches based on stx.9.0 for ansible-playbooks repo: https://opendev.org/starlingx/ansible-playbooks.git Issue-ID: INF-481 Signed-off-by: Jackie Huang Change-Id: Id03d6acec2bdbbd16ba675b682a4bcd6306a67ce --- diff --git a/scripts/build_inf_debian/meta-patches-arm/stx90/cgcs-root/stx/ansible-playbooks/0001-playbooks-fix-images-for-arm64.patch b/scripts/build_inf_debian/meta-patches-arm/stx90/cgcs-root/stx/ansible-playbooks/0001-playbooks-fix-images-for-arm64.patch new file mode 100644 index 00000000..2e4a36ca --- /dev/null +++ b/scripts/build_inf_debian/meta-patches-arm/stx90/cgcs-root/stx/ansible-playbooks/0001-playbooks-fix-images-for-arm64.patch @@ -0,0 +1,285 @@ +From b10fce0a79dfc6bcb9996b59ff78b07be6f715ab Mon Sep 17 00:00:00 2001 +From: Jackie Huang +Date: Fri, 28 Jul 2023 22:15:23 +0800 +Subject: [PATCH 1/2] playbooks: fix images for arm64 + +* Set fact image_architecture and use to replace the + hard coded image arch names. +* Several images don't have arm64 version on the original + registry, use the ones on docker.io/stx4arm for now + +Test Plan: +PASS: downloader -s -b on x86-64 host +PASS: downloader -s -b on arm64 host +PASS: build-pkgs on x86-64 host +PASS: build-image on x86-64 host +PASS: build-pkgs on arm64 host +PASS: build-image on arm64 host +PASS: Deploy AIO-SX on x86-64 target +PASS: Deploy AIO-SX on arm64 target +PASS: Deploy AIO-DX on x86-64 target +PASS: Deploy AIO-DX on arm64 target + +Story: 2010739 +Task: 48020 + +Change-Id: Id3a75f34aeada68c65e9db01ef7140aaa81b8748 +Signed-off-by: Jackie Huang +--- + .../templates/k8s-v1.24.4/multus-cni.yaml.j2 | 4 ++-- + .../templates/k8s-v1.24.4/sriov-cni.yaml.j2 | 4 ++-- + .../templates/k8s-v1.24.4/sriov-plugin.yaml.j2 | 4 ++-- + .../templates/k8s-v1.26.1/multus-cni.yaml.j2 | 4 ++-- + .../templates/k8s-v1.26.1/sriov-cni.yaml.j2 | 4 ++-- + .../templates/k8s-v1.26.1/sriov-plugin.yaml.j2 | 4 ++-- + .../common/load-images-information/tasks/main.yml | 15 +++++++++++++++ + .../vars/k8s-v1.24.4/system-images.yml | 5 ++++- + .../vars/k8s-v1.25.3/system-images.yml | 5 ++++- + .../vars/k8s-v1.26.1/system-images.yml | 5 ++++- + 10 files changed, 39 insertions(+), 15 deletions(-) + +diff --git a/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.24.4/multus-cni.yaml.j2 b/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.24.4/multus-cni.yaml.j2 +index e0adf105..56ac9ca5 100644 +--- a/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.24.4/multus-cni.yaml.j2 ++++ b/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.24.4/multus-cni.yaml.j2 +@@ -200,7 +200,7 @@ data: + apiVersion: apps/v1 + kind: DaemonSet + metadata: +- name: kube-multus-ds-amd64 ++ name: kube-multus-ds + namespace: kube-system + labels: + tier: node +@@ -223,7 +223,7 @@ spec: + spec: + hostNetwork: true + nodeSelector: +- kubernetes.io/arch: amd64 ++ kubernetes.io/arch: "{{ image_architecture }}" + tolerations: + - operator: Exists + effect: NoSchedule +diff --git a/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.24.4/sriov-cni.yaml.j2 b/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.24.4/sriov-cni.yaml.j2 +index d05c3058..c0e14d4a 100644 +--- a/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.24.4/sriov-cni.yaml.j2 ++++ b/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.24.4/sriov-cni.yaml.j2 +@@ -25,7 +25,7 @@ + apiVersion: apps/v1 + kind: DaemonSet + metadata: +- name: kube-sriov-cni-ds-amd64 ++ name: kube-sriov-cni-ds + namespace: kube-system + labels: + tier: node +@@ -47,7 +47,7 @@ spec: + app: sriov-cni + spec: + nodeSelector: +- kubernetes.io/arch: amd64 ++ kubernetes.io/arch: "{{ image_architecture }}" + tolerations: + - operator: Exists + effect: NoSchedule +diff --git a/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.24.4/sriov-plugin.yaml.j2 b/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.24.4/sriov-plugin.yaml.j2 +index 086d2a8a..1744ca2f 100644 +--- a/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.24.4/sriov-plugin.yaml.j2 ++++ b/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.24.4/sriov-plugin.yaml.j2 +@@ -36,7 +36,7 @@ metadata: + apiVersion: apps/v1 + kind: DaemonSet + metadata: +- name: kube-sriov-device-plugin-amd64 ++ name: kube-sriov-device-plugin + namespace: kube-system + labels: + tier: node +@@ -59,7 +59,7 @@ spec: + spec: + hostNetwork: true + nodeSelector: +- beta.kubernetes.io/arch: amd64 ++ kubernetes.io/arch: "{{ image_architecture }}" + sriovdp: enabled + tolerations: + - operator: Exists +diff --git a/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.26.1/multus-cni.yaml.j2 b/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.26.1/multus-cni.yaml.j2 +index 62eba417..8025a531 100644 +--- a/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.26.1/multus-cni.yaml.j2 ++++ b/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.26.1/multus-cni.yaml.j2 +@@ -200,7 +200,7 @@ data: + apiVersion: apps/v1 + kind: DaemonSet + metadata: +- name: kube-multus-ds-amd64 ++ name: kube-multus-ds + namespace: kube-system + labels: + tier: node +@@ -223,7 +223,7 @@ spec: + spec: + hostNetwork: true + nodeSelector: +- kubernetes.io/arch: amd64 ++ kubernetes.io/arch: "{{ image_architecture }}" + tolerations: + - operator: Exists + effect: NoSchedule +diff --git a/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.26.1/sriov-cni.yaml.j2 b/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.26.1/sriov-cni.yaml.j2 +index 74eea969..6306dc73 100644 +--- a/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.26.1/sriov-cni.yaml.j2 ++++ b/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.26.1/sriov-cni.yaml.j2 +@@ -25,7 +25,7 @@ + apiVersion: apps/v1 + kind: DaemonSet + metadata: +- name: kube-sriov-cni-ds-amd64 ++ name: kube-sriov-cni-ds + namespace: kube-system + labels: + tier: node +@@ -47,7 +47,7 @@ spec: + app: sriov-cni + spec: + nodeSelector: +- kubernetes.io/arch: amd64 ++ kubernetes.io/arch: "{{ image_architecture }}" + tolerations: + - operator: Exists + effect: NoSchedule +diff --git a/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.26.1/sriov-plugin.yaml.j2 b/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.26.1/sriov-plugin.yaml.j2 +index 7bf73e26..c7e3b108 100644 +--- a/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.26.1/sriov-plugin.yaml.j2 ++++ b/playbookconfig/src/playbooks/roles/common/bringup-kubemaster/templates/k8s-v1.26.1/sriov-plugin.yaml.j2 +@@ -36,7 +36,7 @@ metadata: + apiVersion: apps/v1 + kind: DaemonSet + metadata: +- name: kube-sriov-device-plugin-amd64 ++ name: kube-sriov-device-plugin + namespace: kube-system + labels: + tier: node +@@ -59,7 +59,7 @@ spec: + spec: + hostNetwork: true + nodeSelector: +- beta.kubernetes.io/arch: amd64 ++ kubernetes.io/arch: "{{ image_architecture }}" + sriovdp: enabled + tolerations: + - operator: Exists +diff --git a/playbookconfig/src/playbooks/roles/common/load-images-information/tasks/main.yml b/playbookconfig/src/playbooks/roles/common/load-images-information/tasks/main.yml +index e8d65c44..3705bce4 100644 +--- a/playbookconfig/src/playbooks/roles/common/load-images-information/tasks/main.yml ++++ b/playbookconfig/src/playbooks/roles/common/load-images-information/tasks/main.yml +@@ -95,6 +95,21 @@ + when: additional_config_file.stat.exists and + inventory_hostname != "localhost" + ++ - name: Retrieve architecture info ++ command: dpkg --print-architecture ++ register: os_architecture ++ ++ - name: set image architecture ++ set_fact: ++ image_architecture: "{{ os_architecture.stdout }}" ++ ++ - name: set images for arm64 ++ set_fact: ++ kubernetes_entrypoint_img: "{{ kubernetes_entrypoint_img_arm64 }}" ++ sriov_cni_img: "{{ sriov_cni_img_arm64 }}" ++ sriov_network_device_img: "{{ sriov_network_device_img_arm64 }}" ++ when: image_architecture == "arm64" ++ + - name: Categorize system images + set_fact: + networking_images: +diff --git a/playbookconfig/src/playbooks/roles/common/load-images-information/vars/k8s-v1.24.4/system-images.yml b/playbookconfig/src/playbooks/roles/common/load-images-information/vars/k8s-v1.24.4/system-images.yml +index 8cecfd2a..cb55604e 100644 +--- a/playbookconfig/src/playbooks/roles/common/load-images-information/vars/k8s-v1.24.4/system-images.yml ++++ b/playbookconfig/src/playbooks/roles/common/load-images-information/vars/k8s-v1.24.4/system-images.yml +@@ -2,17 +2,20 @@ + # System images that are pre-pulled and pushed to local registry + n3000_opae_img: docker.io/starlingx/n3000-opae:stx.8.0-v1.0.2 + kubernetes_entrypoint_img: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 ++kubernetes_entrypoint_img_arm64: docker.io/stx4arm/kubernetes-entrypoint:v0.3.1 + calico_cni_img: quay.io/calico/cni:v3.24.0 + calico_node_img: quay.io/calico/node:v3.24.0 + calico_kube_controllers_img: quay.io/calico/kube-controllers:v3.24.0 + multus_img: ghcr.io/k8snetworkplumbingwg/multus-cni:v3.9.2 + sriov_cni_img: ghcr.io/k8snetworkplumbingwg/sriov-cni:v2.6.3 ++sriov_cni_img_arm64: docker.io/stx4arm/sriov-cni:v2.6.3 + sriov_network_device_img: ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:v3.5.1 ++sriov_network_device_img_arm64: docker.io/stx4arm/sriov-network-device-plugin:v3.5.1 + # Nginx images + nginx_ingress_controller_img: registry.k8s.io/ingress-nginx/controller:v1.9.3 + nginx_kube_webhook_certgen_img: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20231011-8b53cabe0 + nginx_opentelemetry_img: registry.k8s.io/ingress-nginx/opentelemetry:v20230721-3e2062ee5 +-default_backend_img: registry.k8s.io/defaultbackend-amd64:1.5 ++default_backend_img: "registry.k8s.io/defaultbackend-{{ image_architecture }}:1.5" + # Cert-manager images + cert_manager_acmesolver_img: quay.io/jetstack/cert-manager-acmesolver:v1.13.1 + cert_manager_cainjector_img: quay.io/jetstack/cert-manager-cainjector:v1.13.1 +diff --git a/playbookconfig/src/playbooks/roles/common/load-images-information/vars/k8s-v1.25.3/system-images.yml b/playbookconfig/src/playbooks/roles/common/load-images-information/vars/k8s-v1.25.3/system-images.yml +index 2b5b8367..68b23d62 100644 +--- a/playbookconfig/src/playbooks/roles/common/load-images-information/vars/k8s-v1.25.3/system-images.yml ++++ b/playbookconfig/src/playbooks/roles/common/load-images-information/vars/k8s-v1.25.3/system-images.yml +@@ -2,12 +2,15 @@ + # System images that are pre-pulled and pushed to local registry + n3000_opae_img: docker.io/starlingx/n3000-opae:stx.8.0-v1.0.2 + kubernetes_entrypoint_img: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 ++kubernetes_entrypoint_img_arm64: docker.io/stx4arm/kubernetes-entrypoint:v0.3.1 + calico_cni_img: quay.io/calico/cni:v3.25.0 + calico_node_img: quay.io/calico/node:v3.25.0 + calico_kube_controllers_img: quay.io/calico/kube-controllers:v3.25.0 + multus_img: ghcr.io/k8snetworkplumbingwg/multus-cni:v3.9.3 + sriov_cni_img: ghcr.io/k8snetworkplumbingwg/sriov-cni:v2.7.0 ++sriov_cni_img_arm64: docker.io/stx4arm/sriov-cni:v2.7.0 + sriov_network_device_img: ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:v3.5.1 ++sriov_network_device_img_arm64: docker.io/stx4arm/sriov-network-device-plugin:v3.5.1 + intel_qat_plugin_img: docker.io/intel/intel-qat-plugin:0.26.0 + intel_gpu_plugin_img: docker.io/intel/intel-gpu-plugin:0.26.0 + intel_gpu_initcontainer_img: docker.io/intel/intel-gpu-initcontainer:0.26.0 +@@ -15,7 +18,7 @@ intel_gpu_initcontainer_img: docker.io/intel/intel-gpu-initcontainer:0.26.0 + nginx_ingress_controller_img: registry.k8s.io/ingress-nginx/controller:v1.9.3 + nginx_kube_webhook_certgen_img: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20231011-8b53cabe0 + nginx_opentelemetry_img: registry.k8s.io/ingress-nginx/opentelemetry:v20230721-3e2062ee5 +-default_backend_img: registry.k8s.io/defaultbackend-amd64:1.5 ++default_backend_img: "registry.k8s.io/defaultbackend-{{ image_architecture }}:1.5" + # Cert-manager images + cert_manager_acmesolver_img: quay.io/jetstack/cert-manager-acmesolver:v1.13.1 + cert_manager_cainjector_img: quay.io/jetstack/cert-manager-cainjector:v1.13.1 +diff --git a/playbookconfig/src/playbooks/roles/common/load-images-information/vars/k8s-v1.26.1/system-images.yml b/playbookconfig/src/playbooks/roles/common/load-images-information/vars/k8s-v1.26.1/system-images.yml +index 0a9f7584..d536b23e 100644 +--- a/playbookconfig/src/playbooks/roles/common/load-images-information/vars/k8s-v1.26.1/system-images.yml ++++ b/playbookconfig/src/playbooks/roles/common/load-images-information/vars/k8s-v1.26.1/system-images.yml +@@ -2,12 +2,15 @@ + # System images that are pre-pulled and pushed to local registry + n3000_opae_img: docker.io/starlingx/n3000-opae:stx.8.0-v1.0.2 + kubernetes_entrypoint_img: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 ++kubernetes_entrypoint_img_arm64: docker.io/stx4arm/kubernetes-entrypoint:v0.3.1 + calico_cni_img: quay.io/calico/cni:v3.25.0 + calico_node_img: quay.io/calico/node:v3.25.0 + calico_kube_controllers_img: quay.io/calico/kube-controllers:v3.25.0 + multus_img: ghcr.io/k8snetworkplumbingwg/multus-cni:v3.9.3 + sriov_cni_img: ghcr.io/k8snetworkplumbingwg/sriov-cni:v2.7.0 ++sriov_cni_img_arm64: docker.io/stx4arm/sriov-cni:v2.7.0 + sriov_network_device_img: ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:v3.5.1 ++sriov_network_device_img_arm64: docker.io/stx4arm/sriov-network-device-plugin:v3.5.1 + intel_qat_plugin_img: docker.io/intel/intel-qat-plugin:0.26.0 + intel_gpu_plugin_img: docker.io/intel/intel-gpu-plugin:0.26.0 + intel_gpu_initcontainer_img: docker.io/intel/intel-gpu-initcontainer:0.26.0 +@@ -15,7 +18,7 @@ intel_gpu_initcontainer_img: docker.io/intel/intel-gpu-initcontainer:0.26.0 + nginx_ingress_controller_img: registry.k8s.io/ingress-nginx/controller:v1.9.3 + nginx_kube_webhook_certgen_img: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20231011-8b53cabe0 + nginx_opentelemetry_img: registry.k8s.io/ingress-nginx/opentelemetry:v20230721-3e2062ee5 +-default_backend_img: registry.k8s.io/defaultbackend-amd64:1.5 ++default_backend_img: "registry.k8s.io/defaultbackend-{{ image_architecture }}:1.5" + # Cert-manager images + cert_manager_acmesolver_img: quay.io/jetstack/cert-manager-acmesolver:v1.13.1 + cert_manager_cainjector_img: quay.io/jetstack/cert-manager-cainjector:v1.13.1 +-- +2.30.2 + diff --git a/scripts/build_inf_debian/meta-patches-arm/stx90/cgcs-root/stx/ansible-playbooks/0002-download_images-add-support-to-load-image-from-offli.patch b/scripts/build_inf_debian/meta-patches-arm/stx90/cgcs-root/stx/ansible-playbooks/0002-download_images-add-support-to-load-image-from-offli.patch new file mode 100644 index 00000000..922d7526 --- /dev/null +++ b/scripts/build_inf_debian/meta-patches-arm/stx90/cgcs-root/stx/ansible-playbooks/0002-download_images-add-support-to-load-image-from-offli.patch @@ -0,0 +1,107 @@ +From ecdbdc0db662ffb5abb6eca9c84d5307fabad0f6 Mon Sep 17 00:00:00 2001 +From: Jackie Huang +Date: Wed, 19 Jan 2022 04:49:59 -0500 +Subject: [PATCH 2/2] download_images: add support to load image from offline + file + +Add support to load image from offline file, the steps to use +this feature: +1. Get and place all offline image files in a dir: + e.g. /home/sysadmin/docker_images + +2. Add the following line in local.conf: + offline_img_dir: /home/sysadmin/docker_images + +3. run bootstrap playbook as usual. + +Signed-off-by: Jackie Huang +--- + .../files/download_images.py | 28 +++++++++++++++---- + .../common/push-docker-images/tasks/main.yml | 2 ++ + 2 files changed, 25 insertions(+), 5 deletions(-) + +diff --git a/playbookconfig/src/playbooks/roles/common/push-docker-images/files/download_images.py b/playbookconfig/src/playbooks/roles/common/push-docker-images/files/download_images.py +index e4239188..8e47a28b 100644 +--- a/playbookconfig/src/playbooks/roles/common/push-docker-images/files/download_images.py ++++ b/playbookconfig/src/playbooks/roles/common/push-docker-images/files/download_images.py +@@ -43,6 +43,8 @@ registries = json.loads(os.environ['REGISTRIES']) + add_docker_prefix = False + crictl_image_list = [] + ++offline_img_suffix = ['.tar.gz', '.tar.bz2', '.tar'] ++offline_img_dir = os.environ.get('OFFLINE_IMG_DIR', '') + + def get_local_registry_auth(): + password = keyring.get_password("sysinv", "services") +@@ -50,6 +52,16 @@ def get_local_registry_auth(): + raise Exception("Local registry password not found.") + return dict(username="sysinv", password=str(password)) + ++def get_offline_img_file(img): ++ if not offline_img_dir: ++ return False ++ ++ for suffix in offline_img_suffix: ++ img_file_name = img.replace("/", "_").replace(":", "_") + suffix ++ img_file = os.path.join(offline_img_dir, img_file_name) ++ if os.path.exists(img_file): ++ return img_file ++ return False + + def convert_img_for_local_lookup(img): + # This function converts the given image reference to the +@@ -178,6 +190,7 @@ def download_and_push_an_image(img): + local_img = convert_img_for_local_lookup(img) + target_img = get_img_tag_with_registry(img) + err_msg = " Image download failed: %s " % target_img ++ offline_img_file = get_offline_img_file(img) + + client = docker.APIClient() + auth = get_local_registry_auth() +@@ -201,13 +214,18 @@ def download_and_push_an_image(img): + return target_img, True + except docker.errors.APIError as e: + print(str(e)) +- print("Image %s not found on local registry, attempt to download..." ++ print("Image %s not found on local registry, attempt to load from offline file or download..." + % target_img) + try: +- response = client.pull(target_img) +- check_response(response) +- print("Image download succeeded: %s" % target_img) +- client.tag(target_img, local_img) ++ if offline_img_file: ++ with open(offline_img_file, 'rb') as f: ++ client.load_image(f) ++ print("Image loaded from offline file (%s) succeeded: %s" % (offline_img_file, local_img)) ++ else: ++ response = client.pull(target_img) ++ check_response(response) ++ print("Image download succeeded: %s" % target_img) ++ client.tag(target_img, local_img) + client.push(local_img, auth_config=auth) + print("Image push succeeded: %s" % local_img) + +diff --git a/playbookconfig/src/playbooks/roles/common/push-docker-images/tasks/main.yml b/playbookconfig/src/playbooks/roles/common/push-docker-images/tasks/main.yml +index 1429b9c9..be9e9a49 100644 +--- a/playbookconfig/src/playbooks/roles/common/push-docker-images/tasks/main.yml ++++ b/playbookconfig/src/playbooks/roles/common/push-docker-images/tasks/main.yml +@@ -260,6 +260,7 @@ + - block: + - set_fact: + download_images: "{{ download_images_list | join(',') }}" ++ offline_img_dir: "{{ offline_img_dir | default('') }}" + + - name: "{{ download_images_task_name }}" + script: download_images.py {{ download_images }} +@@ -271,6 +272,7 @@ + environment: + REGISTRIES: "{{ registries | to_json }}" + ADD_DOCKER_PREFIX: "{{ add_docker_prefix }}" ++ OFFLINE_IMG_DIR: "{{ offline_img_dir }}" + + - debug: + msg: "{{ download_images_output.stdout_lines }}" +-- +2.30.2 +