[DO-1431] Final config PROD k8s (!3)

DO-1431

Co-authored-by: denis.patrakeev <denis.patrakeev@avroid.tech>
Reviewed-on: https://git.avroid.tech/K8s/k8s-deploy/pulls/3

.gitmodules

@@ -1,4 +1,4 @@
-[submodule "env/avroid_prod/kubespray"]
+[submodule "env/avroid_prod/k8s-avroid-office.prod.local/kubespray"]
-	path = env/avroid_prod/kubespray
+	path = env/avroid_prod/k8s-avroid-office.prod.local/kubespray
 	url = ssh://git@git.avroid.tech:2222/Mirrors/kubespray.git
 	branch = v2.26.0

README.md

@@ -15,40 +15,60 @@ git submodule update --init --recursive
 .
 |-- env - директория содержащая подкаталоги соответствующие различным окружениям 
 |   |-- <ОКРУЖЕНИЕ_01> - директория окружения, название директории совпадает с названием окружения
-|   |   |-- inventory - каталог содержит Ansible inventory к Kubespray для деплоя кластера
+|   |   |-- <КЛАСТЕР_01> - директория кластера, название директории совпадает с именем кластера
-|   |   |
+|   |       |-- inventory - каталог содержит Ansible inventory к Kubespray для деплоя кластера
-|   |   |-- kubespray - каталог является Git Submodules на определённый тег Kubespray
+|   |       |
-|   |   |
+|   |       |-- kubespray - каталог является Git Submodules на определённый тег Kubespray
-|   |   |-- patches - кастомные доработки для отдельных сервисов кластера
+|   |       |
-|   |   |   |-- <ИМЯ_СЕРВИСА_01> - директория, содержащая файлы патчей (без подкаталогов)
+|   |       |-- patches - кастомные доработки для отдельных сервисов кластера
-|   |   |   |   |-- file_XX.yaml - файл с кастомными доработками
+|   |       |   |-- <ИМЯ_СЕРВИСА_01> - директория, содержащая файлы патчей (без подкаталогов)
-|   |   |   |   |-- ...
+|   |       |   |   |-- file_XX.yaml - файл с кастомными доработками
-|   |   |   |
+|   |       |   |   |-- ...
-|   |   |   |-- ...
+|   |       |   |
-|   |   |   |
+|   |       |   |-- ...
-|   |   |   |-- <ИМЯ_СЕРВИСА_XX>
+|   |       |   |
-|   |   |       |-- file_XX.yaml
+|   |       |   |-- <ИМЯ_СЕРВИСА_XX>
-|   |   |       |-- ...
+|   |       |       |-- file_XX.yaml
-|   |   |
+|   |       |       |-- ...
-|   |   |-- README.md - файл содержит подробное описание конфигурации и порядок деплоя
+|   |       |
+|   |       |-- README.md - файл содержит подробное описание конфигурации и порядок деплоя
 |   |    
 |   |-- ...    
 |   |    
 |   |-- <ОКРУЖЕНИЕ_XX>
-|       |-- inventory
+|       |-- <КЛАСТЕР_01>
-|       |-- kubespray
+|       |   |-- inventory
-|       |-- patches
+|       |   |-- kubespray
-|       |   |-- <ИМЯ_СЕРВИСА_01>
+|       |   |-- patches
-|       |   |   |-- file_XX.yaml
+|       |   |   |-- <ИМЯ_СЕРВИСА_01>
+|       |   |   |   |-- file_XX.yaml
+|       |   |   |   |-- ...
+|       |   |   |
 |       |   |   |-- ...
+|       |   |   |
+|       |   |   |-- <ИМЯ_СЕРВИСА_XX>
+|       |   |       |-- file_XX.yaml
+|       |   |       |-- ...
 |       |   |
-|       |   |-- ...
+|       |   |-- README.md
-|       |   |
-|       |   |-- <ИМЯ_СЕРВИСА_XX>
-|       |       |-- file_XX.yaml
-|       |       |-- ...
 |       |
-|       |-- README.md
+|       |-- ...
+|       |
+|       |-- <КЛАСТЕР_XX>
+|           |-- inventory
+|           |-- kubespray
+|           |-- patches
+|           |   |-- <ИМЯ_СЕРВИСА_01>
+|           |   |   |-- file_XX.yaml
+|           |   |   |-- ...
+|           |   |
+|           |   |-- ...
+|           |   |
+|           |   |-- <ИМЯ_СЕРВИСА_XX>
+|           |       |-- file_XX.yaml
+|           |       |-- ...
+|           |
+|           |-- README.md
 |
 |-- .gitignore
 |-- README.md
@@ -58,16 +78,16 @@ git submodule update --init --recursive
 
 Сначала создаём Git Submodule:
 ```bash
-cd env/<ОКРУЖЕНИЕ_XX>
+cd env/<ОКРУЖЕНИЕ_XX>/<КЛАСТЕР_XX>
 git submodule add ssh://git@git.avroid.tech:2222/Mirrors/kubespray.git kubespray
 ```
 
 После чего принудительно переключаем Git Submodule на нужный тэг (релиз) Kubespray:
 ```bash
-cd env/<ОКРУЖЕНИЕ_XX>/kubespray
+cd env/<ОКРУЖЕНИЕ_XX>/<КЛАСТЕР_XX>/kubespray
 git checkout v2.26.0
-cd ../../..
+cd ../../../..
-git add env/<ОКРУЖЕНИЕ_XX>/kubespray
+git add env/<ОКРУЖЕНИЕ_XX>/<КЛАСТЕР_XX>/kubespray
 ```
 
 После чего правим файл `.gitmodules` и явно в нём прописываем необходимый тэг,
@@ -75,8 +95,8 @@ git add env/<ОКРУЖЕНИЕ_XX>/kubespray
 
 Пример записи:
 ```text
-[submodule "env/<ОКРУЖЕНИЕ_ХХ>/kubespray"]
+[submodule "env/<ОКРУЖЕНИЕ_ХХ>/<КЛАСТЕР_XX>/kubespray"]
-	path = env/<ОКРУЖЕНИЕ_ХХ>/kubespray
+	path = env/<ОКРУЖЕНИЕ_ХХ>/<КЛАСТЕР_XX>/kubespray
 	url = ssh://git@git.avroid.tech:2222/Mirrors/kubespray.git
 	branch = v2.26.0
 ```

env/avroid_prod/inventory/group_vars/all/offline.yml

@@ -1,116 +0,0 @@
----
-## Global Offline settings
-### Private Container Image Registry
-# registry_host: "myprivateregisry.com"
-# files_repo: "http://myprivatehttpd"
-### If using CentOS, RedHat, AlmaLinux or Fedora
-# yum_repo: "http://myinternalyumrepo"
-### If using Debian
-# debian_repo: "http://myinternaldebianrepo"
-### If using Ubuntu
-# ubuntu_repo: "http://myinternalubunturepo"
-
-## Container Registry overrides
-# kube_image_repo: "{{ registry_host }}"
-# gcr_image_repo: "{{ registry_host }}"
-# github_image_repo: "{{ registry_host }}"
-# docker_image_repo: "{{ registry_host }}"
-# quay_image_repo: "{{ registry_host }}"
-
-## Kubernetes components
-# kubeadm_download_url: "{{ files_repo }}/dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubeadm"
-# kubectl_download_url: "{{ files_repo }}/dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl"
-# kubelet_download_url: "{{ files_repo }}/dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet"
-
-
-## Two options - Override entire repository or override only a single binary.
-
-## [Optional] 1 - Override entire binary repository
-# github_url: "https://my_github_proxy"
-# dl_k8s_io_url: "https://my_dl_k8s_io_proxy"
-# storage_googleapis_url: "https://my_storage_googleapi_proxy"
-# get_helm_url: "https://my_helm_sh_proxy"
-
-## [Optional] 2 - Override a specific binary
-## CNI Plugins
-# cni_download_url: "{{ files_repo }}/github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
-
-## cri-tools
-# crictl_download_url: "{{ files_repo }}/github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
-
-## [Optional] etcd: only if you use etcd_deployment=host
-# etcd_download_url: "{{ files_repo }}/github.com/etcd-io/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
-
-# [Optional] Calico: If using Calico network plugin
-# calicoctl_download_url: "{{ files_repo }}/github.com/projectcalico/calico/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
-# [Optional] Calico with kdd: If using Calico network plugin with kdd datastore
-# calico_crds_download_url: "{{ files_repo }}/github.com/projectcalico/calico/archive/{{ calico_version }}.tar.gz"
-
-# [Optional] Cilium: If using Cilium network plugin
-# ciliumcli_download_url: "{{ files_repo }}/github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_version }}/cilium-linux-{{ image_arch }}.tar.gz"
-
-# [Optional] helm: only if you set helm_enabled: true
-# helm_download_url: "{{ files_repo }}/get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
-
-# [Optional] crun: only if you set crun_enabled: true
-# crun_download_url: "{{ files_repo }}/github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}"
-
-# [Optional] kata: only if you set kata_containers_enabled: true
-# kata_containers_download_url: "{{ files_repo }}/github.com/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ ansible_architecture }}.tar.xz"
-
-# [Optional] cri-dockerd: only if you set container_manager: docker
-# cri_dockerd_download_url: "{{ files_repo }}/github.com/Mirantis/cri-dockerd/releases/download/v{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tgz"
-
-# [Optional] runc: if you set container_manager to containerd or crio
-# runc_download_url: "{{ files_repo }}/github.com/opencontainers/runc/releases/download/{{ runc_version }}/runc.{{ image_arch }}"
-
-# [Optional] cri-o: only if you set container_manager: crio
-# crio_download_base: "download.opensuse.org/repositories/devel:kubic:libcontainers:stable"
-# crio_download_crio: "http://{{ crio_download_base }}:/cri-o:/"
-# crio_download_url: "{{ files_repo }}/storage.googleapis.com/cri-o/artifacts/cri-o.{{ image_arch }}.{{ crio_version }}.tar.gz"
-# skopeo_download_url: "{{ files_repo }}/github.com/lework/skopeo-binary/releases/download/{{ skopeo_version }}/skopeo-linux-{{ image_arch }}"
-
-# [Optional] containerd: only if you set container_runtime: containerd
-# containerd_download_url: "{{ files_repo }}/github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
-# nerdctl_download_url: "{{ files_repo }}/github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
-
-# [Optional] runsc,containerd-shim-runsc: only if you set gvisor_enabled: true
-# gvisor_runsc_download_url: "{{ files_repo }}/storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/runsc"
-# gvisor_containerd_shim_runsc_download_url: "{{ files_repo }}/storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/containerd-shim-runsc-v1"
-
-# [Optional] Krew: only if you set krew_enabled: true
-# krew_download_url: "{{ files_repo }}/github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz"
-
-## CentOS/Redhat/AlmaLinux
-### For EL8, baseos and appstream must be available,
-### By default we enable those repo automatically
-# rhel_enable_repos: false
-### Docker / Containerd
-# docker_rh_repo_base_url: "{{ yum_repo }}/docker-ce/$releasever/$basearch"
-# docker_rh_repo_gpgkey: "{{ yum_repo }}/docker-ce/gpg"
-
-## Fedora
-### Docker
-# docker_fedora_repo_base_url: "{{ yum_repo }}/docker-ce/{{ ansible_distribution_major_version }}/{{ ansible_architecture }}"
-# docker_fedora_repo_gpgkey: "{{ yum_repo }}/docker-ce/gpg"
-### Containerd
-# containerd_fedora_repo_base_url: "{{ yum_repo }}/containerd"
-# containerd_fedora_repo_gpgkey: "{{ yum_repo }}/docker-ce/gpg"
-
-## Debian
-### Docker
-# docker_debian_repo_base_url: "{{ debian_repo }}/docker-ce"
-# docker_debian_repo_gpgkey: "{{ debian_repo }}/docker-ce/gpg"
-### Containerd
-# containerd_debian_repo_base_url: "{{ debian_repo }}/containerd"
-# containerd_debian_repo_gpgkey: "{{ debian_repo }}/containerd/gpg"
-# containerd_debian_repo_repokey: 'YOURREPOKEY'
-
-## Ubuntu
-### Docker
-# docker_ubuntu_repo_base_url: "{{ ubuntu_repo }}/docker-ce"
-# docker_ubuntu_repo_gpgkey: "{{ ubuntu_repo }}/docker-ce/gpg"
-### Containerd
-# containerd_ubuntu_repo_base_url: "{{ ubuntu_repo }}/containerd"
-# containerd_ubuntu_repo_gpgkey: "{{ ubuntu_repo }}/containerd/gpg"
-# containerd_ubuntu_repo_repokey: 'YOURREPOKEY'

env/avroid_prod/k8s-avroid-office.prod.local/README.md

@@ -10,7 +10,7 @@
 ## Особенности развертывания кластера
 | Модуль                   | Комментарий                                                                              |
 |--------------------------|------------------------------------------------------------------------------------------|
-| Cluster name             | k8s.prod.local                                                                           |
+| Cluster name             | k8s-avroid-office.prod.local                                                             |
 | Сеть                     | Только IPv4                                                                              |
 | Сеть                     | 172.24.0.0/18 - подсеть сервисов                                                         |
 | Сеть                     | 172.24.64.0/18 - подсеть подов                                                           |
@@ -18,7 +18,7 @@
 | Маска подсети на ноду    | 24 (Итого - max 254 подов на ноде и max 64 ноды)                                         |
 | CNI                      | calico                                                                                   |
 | NTP-клиенты              | Настроены на локальные приватные NTP-сервера и московскую таймзону                       |
-| DNS zone                 | k8s.prod.local                                                                           |
+| DNS zone                 | k8s-avroid-office.prod.local                                                             |
 | DNS                      | Dual CoreDNS + nodelocaldns                                                              |
 | Etcd                     | данные сервиса в /data/etcd на отдельном блочном устройстве с ext4)                      |
 | Container runtime        | containerd (/var/lib/containerd на отдельном блочном устройстве с XFS)                   |
@@ -28,6 +28,7 @@
 | Диски                    | k8s-worker/build-0X: /var/lib/kubelet/pods вынесен на отдельные блочное устройства с XFS |
 | HA                       | API Server                                                                               |
 | Ingress                  | Nginx ingress controller 80 --> 30080 (k8s-worker-0X), 443 --> 30081 (k8s-worker-0X)     |
+| Ingress                  | Работает только на нодах с кастомной меткой `node-role.kubernetes.io/ingress-nginx:true` |
 | Дополнительные сервисы   | Helm, Metrics Server, Cert manager, netchecker                                           |
 
 
@@ -61,11 +62,11 @@ http://<IP_АДРЕС_НОДЫ>:31081/metrics
 
 ### 2. Обновляем подмодуль с Kubespray и проверяем что он стоит на необходимом тэге
 ```bash
-cd env/<ОКРУЖЕНИЕ_XX>
+cd env/<ОКРУЖЕНИЕ_XX>/<КЛАСТЕР_XX>
 git submodule update --init --recursive
 cd kubespray
 git status
-cd ../..
+cd ../../..
 ```
 
 ### 3. Готовим окружение Ansible
@@ -78,7 +79,7 @@ cd ../..
 | >=2.16.4        | 3.10-3.12      |
 
 ```bash
-cd env/<ОКРУЖЕНИЕ_XX>
+cd env/<ОКРУЖЕНИЕ_XX>/<КЛАСТЕР_XX>
 export VENVDIR=kubespray-venv
 export KUBESPRAYDIR=kubespray
 python3 -m venv ./$VENVDIR
@@ -88,7 +89,7 @@ pip3 install -U -r $KUBESPRAYDIR/requirements.txt
 
 ### 4. Запускаем раскатку кластера
 ```bash
-cd env/<ОКРУЖЕНИЕ_XX>
+cd env/<ОКРУЖЕНИЕ_XX>/<КЛАСТЕР_XX>
 export VENVDIR=kubespray-venv
 export KUBESPRAYDIR=kubespray
 source $VENVDIR/bin/activate

env/avroid_prod/k8s-avroid-office.prod.local/inventory/credentials/kubeadm_certificate_key.creds

@@ -0,0 +1 @@
+bcdF76b6B3F3cBE15afe5eea979e9c8056dFBF5c14ce9e71eC414413bDfCA0DA

env/avroid_prod/k8s-avroid-office.prod.local/inventory/group_vars/all/all.yml

@@ -19,7 +19,7 @@ bin_dir: /usr/local/bin
 ## Internal loadbalancers for apiservers
 loadbalancer_apiserver_localhost: true
 # valid options are "nginx" or "haproxy"
-loadbalancer_apiserver_type: nginx
+loadbalancer_apiserver_type: nginx  # valid values "nginx" or "haproxy"
 
 ## Local loadbalancer should use this port
 ## And must be set port 6443

env/avroid_prod/k8s-avroid-office.prod.local/inventory/group_vars/custom_kube_node_with_ingress/custom_vars.yml

@@ -0,0 +1,3 @@
+---
+node_labels:
+  node-role.kubernetes.io/ingress-nginx: "true"

env/avroid_prod/k8s-avroid-office.prod.local/inventory/group_vars/k8s_cluster/addons.yml

@@ -108,9 +108,7 @@ ingress_nginx_service_nodeport_http: 30080
 ingress_nginx_service_nodeport_https: 30081
 ingress_publish_status_address: ""
 ingress_nginx_nodeselector:
-  - kubernetes.io/hostname: "k8s-worker-01"
+  node-role.kubernetes.io/ingress-nginx: "true"
-  - kubernetes.io/hostname: "k8s-worker-02"
-  - kubernetes.io/hostname: "k8s-worker-03"
 ingress_nginx_tolerations:
   - key: "node-role.kubernetes.io/control-node"
     operator: "Equal"

env/avroid_prod/k8s-avroid-office.prod.local/inventory/group_vars/k8s_cluster/k8s-cluster.yml

@@ -157,7 +157,7 @@ kube_encrypt_secret_data: false
 
 # DNS configuration.
 # Kubernetes cluster name, also will be used as DNS domain
-cluster_name: k8s.prod.local
+cluster_name: k8s-avroid-office.prod.local
 # Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods
 ndots: 2
 # dns_timeout: 2
@@ -339,12 +339,29 @@ persistent_volumes_enabled: false
 tls_min_version: "VersionTLS12"
 
 ## Support tls cipher suites.
-tls_cipher_suites:
+# tls_cipher_suites: {}
-  - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA
-  - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
-  - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
-  - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
-  - TLS_RSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA
+#   - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+#   - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+#   - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
+#   - TLS_ECDHE_RSA_WITH_RC4_128_SHA
+#   - TLS_RSA_WITH_3DES_EDE_CBC_SHA
+#   - TLS_RSA_WITH_AES_128_CBC_SHA
+#   - TLS_RSA_WITH_AES_128_CBC_SHA256
+#   - TLS_RSA_WITH_AES_128_GCM_SHA256
+#   - TLS_RSA_WITH_AES_256_CBC_SHA
+#   - TLS_RSA_WITH_AES_256_GCM_SHA384
+#   - TLS_RSA_WITH_RC4_128_SHA
 
 ## Amount of time to retain events. (default 1h0m0s)
 event_ttl_duration: "1h0m0s"

env/avroid_prod/k8s-avroid-office.prod.local/inventory/inventory.ini

@@ -30,10 +30,12 @@ k8s-control-01
 k8s-control-02
 k8s-control-03
 
-[kube_node]
+[custom_kube_node_with_ingress]
 k8s-worker-01
 k8s-worker-02
 k8s-worker-03
+
+[kube_node]
 k8s-build-01
 k8s-build-02
 k8s-build-03
@@ -42,6 +44,9 @@ k8s-build-05
 k8s-build-06
 k8s-build-07
 
+[kube_node:children]
+custom_kube_node_with_ingress
+
 #[calico_rr]
 
 [k8s_cluster:children]

env/avroid_prod/k8s-avroid-office.prod.local/namespaces/sandbox/sandbox-namespace.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: sandbox
+  labels:
+    name: sandbox
+    app.kubernetes.io/managed-by: manual

env/avroid_prod/k8s-avroid-office.prod.local/namespaces/sandbox/sandbox-resourcequota.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+  name: sandbox
+  labels:
+    app.kubernetes.io/managed-by: manual
+spec:
+  hard:
+    requests.cpu: "8"
+    requests.memory: 24Gi
+    limits.cpu: "16"
+    limits.memory: 32Gi

example/namespaces/example/example-namespace.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: example
+  labels:
+    name: example
+    app.kubernetes.io/managed-by: manual

example/namespaces/example/example-networkpolicy.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: example
+  namespace: example
+  labels:
+    app.kubernetes.io/managed-by: manual
+spec:
+  podSelector: {}
+  policyTypes:
+  - Ingress
+  - Egress

example/namespaces/example/example-resourcequota.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+  name: example
+  labels:
+    app.kubernetes.io/managed-by: manual
+spec:
+  hard:
+    requests.cpu: "8"
+    requests.memory: 24Gi
+    limits.cpu: "16"
+    limits.memory: 32Gi