[DO-143] Final prepare for env k8s avroid_prod (!2)

DO-1431

Co-authored-by: denis.patrakeev <denis.patrakeev@avroid.tech>
Reviewed-on: https://git.avroid.tech/K8s/k8s-deploy/pulls/2

env/avroid_prod/inventory/group_vars/k8s_cluster/addons.yml

@@ -4,7 +4,7 @@
 # dashboard_enabled: false
 
 # Helm deployment
-helm_enabled: false
+helm_enabled: true
 
 # Registry deployment
 registry_enabled: false
@@ -13,13 +13,13 @@ registry_enabled: false
 # registry_disk_size: "10Gi"
 
 # Metrics Server deployment
-metrics_server_enabled: false
-# metrics_server_container_port: 10250
-# metrics_server_kubelet_insecure_tls: true
-# metrics_server_metric_resolution: 15s
-# metrics_server_kubelet_preferred_address_types: "InternalIP,ExternalIP,Hostname"
-# metrics_server_host_network: false
-# metrics_server_replicas: 1
+metrics_server_enabled: true
+metrics_server_container_port: 10250
+metrics_server_kubelet_insecure_tls: true
+metrics_server_metric_resolution: 15s
+metrics_server_kubelet_preferred_address_types: "InternalIP,ExternalIP,Hostname"
+metrics_server_host_network: false
+metrics_server_replicas: 1
 
 # Rancher Local Path Provisioner
 local_path_provisioner_enabled: false
@@ -96,34 +96,43 @@ rbd_provisioner_enabled: false
 # rbd_provisioner_storage_class: rbd
 # rbd_provisioner_reclaim_policy: Delete
 
+# Gateway API CRDs
+gateway_api_enabled: false
+# gateway_api_experimental_channel: false
+
 # Nginx ingress controller deployment
-ingress_nginx_enabled: false
-# ingress_nginx_host_network: false
-# ingress_nginx_service_type: LoadBalancer
-# ingress_nginx_service_nodeport_http: 30080
-# ingress_nginx_service_nodeport_https: 30081
+ingress_nginx_enabled: true
+ingress_nginx_host_network: false
+ingress_nginx_service_type: NodePort
+ingress_nginx_service_nodeport_http: 30080
+ingress_nginx_service_nodeport_https: 30081
 ingress_publish_status_address: ""
-# ingress_nginx_nodeselector:
-#   kubernetes.io/os: "linux"
-# ingress_nginx_tolerations:
-#   - key: "node-role.kubernetes.io/control-plane"
-#     operator: "Equal"
-#     value: ""
-#     effect: "NoSchedule"
-# ingress_nginx_namespace: "ingress-nginx"
-# ingress_nginx_insecure_port: 80
-# ingress_nginx_secure_port: 443
-# ingress_nginx_configmap:
-#   map-hash-bucket-size: "128"
-#   ssl-protocols: "TLSv1.2 TLSv1.3"
+ingress_nginx_nodeselector:
+  - kubernetes.io/hostname: "k8s-worker-01"
+  - kubernetes.io/hostname: "k8s-worker-02"
+  - kubernetes.io/hostname: "k8s-worker-03"
+ingress_nginx_tolerations:
+  - key: "node-role.kubernetes.io/control-node"
+    operator: "Equal"
+    value: ""
+    effect: "NoSchedule"
+ingress_nginx_namespace: "ingress-nginx"
+ingress_nginx_insecure_port: 80
+ingress_nginx_secure_port: 443
+ingress_nginx_configmap:
+  map-hash-bucket-size: "128"
+  ssl-protocols: "TLSv1.2 TLSv1.3"
+  client-body-buffer-size: "50m"
+  proxy-body-size: "100m"
+  client-header-buffer-size: "2k"
 # ingress_nginx_configmap_tcp_services:
 #   9000: "default/example-go:8080"
 # ingress_nginx_configmap_udp_services:
 #   53: "kube-system/coredns:53"
 # ingress_nginx_extra_args:
 #   - --default-ssl-certificate=default/foo-tls
-# ingress_nginx_termination_grace_period_seconds: 300
-# ingress_nginx_class: nginx
+ingress_nginx_termination_grace_period_seconds: 300
+ingress_nginx_class: nginx
 # ingress_nginx_without_class: true
 # ingress_nginx_default: false
 
@@ -136,23 +145,23 @@ ingress_alb_enabled: false
 # alb_ingress_aws_debug: "false"
 
 # Cert manager deployment
-cert_manager_enabled: false
-# cert_manager_namespace: "cert-manager"
-# cert_manager_tolerations:
-#   - key: node-role.kubernetes.io/control-plane
-#     effect: NoSchedule
-# cert_manager_affinity:
-#  nodeAffinity:
-#    preferredDuringSchedulingIgnoredDuringExecution:
-#    - weight: 100
-#      preference:
-#        matchExpressions:
-#        - key: node-role.kubernetes.io/control-plane
-#          operator: In
-#          values:
-#          - ""
-# cert_manager_nodeselector:
-#   kubernetes.io/os: "linux"
+cert_manager_enabled: true
+cert_manager_namespace: "cert-manager"
+cert_manager_tolerations:
+  - key: node-role.kubernetes.io/control-plane
+    effect: NoSchedule
+cert_manager_affinity:
+  nodeAffinity:
+    preferredDuringSchedulingIgnoredDuringExecution:
+    - weight: 100
+      preference:
+        matchExpressions:
+        - key: node-role.kubernetes.io/control-plane
+          operator: In
+          values:
+          - ""
+cert_manager_nodeselector:
+  kubernetes.io/os: "linux"
 
 # cert_manager_trusted_internal_ca: |
 #   -----BEGIN CERTIFICATE-----
@@ -249,7 +258,7 @@ argocd_enabled: false
 # argocd_admin_password: "password"
 
 # The plugin manager for kubectl
-krew_enabled: false
+krew_enabled: true
 krew_root_dir: "/usr/local/krew"
 
 # Kube VIP

env/avroid_prod/inventory/group_vars/k8s_cluster/k8s-cluster.yml

@@ -17,7 +17,7 @@ kube_token_dir: "{{ kube_config_dir }}/tokens"
 kube_api_anonymous_auth: true
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.29.10
+kube_version: v1.30.4
 
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)
@@ -73,12 +73,12 @@ kube_network_plugin: calico
 kube_network_plugin_multus: false
 
 # Kubernetes internal network for services, unused block of space.
-kube_service_addresses: 10.233.0.0/18
+kube_service_addresses: 172.24.0.0/18
 
 # internal network. When used, it will assign IP
 # addresses from this range to individual pods.
 # This network must be unused in your network infrastructure!
-kube_pods_subnet: 10.233.64.0/18
+kube_pods_subnet: 172.24.64.0/18
 
 # internal network node size allocation (optional). This is the size allocated
 # to each node for pod IP address allocation. Note that the number of pods per node is
@@ -157,7 +157,7 @@ kube_encrypt_secret_data: false
 
 # DNS configuration.
 # Kubernetes cluster name, also will be used as DNS domain
-cluster_name: cluster.local
+cluster_name: k8s.prod.local
 # Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods
 ndots: 2
 # dns_timeout: 2
@@ -169,7 +169,7 @@ ndots: 2
 # Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``).
 # remove_default_searchdomains: false
 # Can be coredns, coredns_dual, manual or none
-dns_mode: coredns
+dns_mode: coredns_dual
 # Set manual server if using a custom cluster DNS server
 # manual_dns_server: 10.x.x.x
 # Enable nodelocal dns cache
@@ -180,26 +180,18 @@ nodelocaldns_health_port: 9254
 nodelocaldns_second_health_port: 9256
 nodelocaldns_bind_metrics_host_ip: false
 nodelocaldns_secondary_skew_seconds: 5
-# nodelocaldns_external_zones:
-# - zones:
-#   - example.com
-#   - example.io:1053
-#   nameservers:
-#   - 1.1.1.1
-#   - 2.2.2.2
-#   cache: 5
-# - zones:
-#   - https://mycompany.local:4453
-#   nameservers:
-#   - 192.168.0.53
-#   cache: 0
-# - zones:
-#   - mydomain.tld
-#   nameservers:
-#   - 10.233.0.3
-#   cache: 5
-#   rewrite:
-#   - name website.tld website.namespace.svc.cluster.local
+nodelocaldns_external_zones:
+  - zones:
+      - avroid.tech
+      - avroid.team
+      - avroid.cloud
+      - adlinux.store
+      - o2linux.org
+    nameservers:
+      - 10.2.4.10
+      - 10.2.4.20
+      - 10.3.0.101
+    cache: 5
 # Enable k8s_external plugin for CoreDNS
 enable_coredns_k8s_external: false
 coredns_k8s_external_zone: k8s_external.local
@@ -214,10 +206,23 @@ enable_coredns_k8s_endpoint_pod_names: false
 # Forward extra domains to the coredns kubernetes plugin
 # coredns_kubernetes_extra_domains: ''
 
+coredns_external_zones:
+  - zones:
+      - avroid.tech
+      - avroid.team
+      - avroid.cloud
+      - adlinux.store
+      - o2linux.org
+    nameservers:
+      - 10.2.4.10
+      - 10.2.4.20
+      - 10.3.0.101
+    cache: 5
+
 # Can be docker_dns, host_resolvconf or none
 resolvconf_mode: host_resolvconf
 # Deploy netchecker app to verify DNS resolve as an HTTP service
-deploy_netchecker: false
+deploy_netchecker: true
 # Ip address of the kubernetes skydns service
 skydns_server: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}"
 skydns_server_secondary: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}"
@@ -248,7 +253,7 @@ default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir"
 # Use ansible_host as external api ip when copying over kubeconfig.
 # kubeconfig_localhost_ansible_host: false
 # Download kubectl onto the host that runs Ansible in {{ bin_dir }}
-# kubectl_localhost: false
+kubectl_localhost: false
 
 # A comma separated list of levels of node allocatable enforcement to be enforced by kubelet.
 # Acceptable options are 'pods', 'system-reserved', 'kube-reserved' and ''. Default is "".
@@ -263,34 +268,34 @@ default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir"
 # kubelet_kubelet_cgroups_cgroupfs: "/system.slice/kubelet.service"
 
 # Optionally reserve this space for kube daemons.
-# kube_reserved: false
+kube_reserved: true
 ## Uncomment to override default values
 ## The following two items need to be set when kube_reserved is true
-# kube_reserved_cgroups_for_service_slice: kube.slice
-# kube_reserved_cgroups: "/{{ kube_reserved_cgroups_for_service_slice }}"
-# kube_memory_reserved: 256Mi
-# kube_cpu_reserved: 100m
-# kube_ephemeral_storage_reserved: 2Gi
+kube_reserved_cgroups_for_service_slice: kube.slice
+kube_reserved_cgroups: "/{{ kube_reserved_cgroups_for_service_slice }}"
+kube_memory_reserved: 256Mi
+kube_cpu_reserved: 100m
+kube_ephemeral_storage_reserved: 2Gi
 # kube_pid_reserved: "1000"
 # Reservation for master hosts
-# kube_master_memory_reserved: 512Mi
-# kube_master_cpu_reserved: 200m
-# kube_master_ephemeral_storage_reserved: 2Gi
+kube_master_memory_reserved: 512Mi
+kube_master_cpu_reserved: 200m
+kube_master_ephemeral_storage_reserved: 2Gi
 # kube_master_pid_reserved: "1000"
 
 ## Optionally reserve resources for OS system daemons.
-# system_reserved: true
+system_reserved: true
 ## Uncomment to override default values
 ## The following two items need to be set when system_reserved is true
-# system_reserved_cgroups_for_service_slice: system.slice
-# system_reserved_cgroups: "/{{ system_reserved_cgroups_for_service_slice }}"
-# system_memory_reserved: 512Mi
-# system_cpu_reserved: 500m
-# system_ephemeral_storage_reserved: 2Gi
+system_reserved_cgroups_for_service_slice: system.slice
+system_reserved_cgroups: "/{{ system_reserved_cgroups_for_service_slice }}"
+system_memory_reserved: 512Mi
+system_cpu_reserved: 500m
+system_ephemeral_storage_reserved: 2Gi
 ## Reservation for master hosts
-# system_master_memory_reserved: 256Mi
-# system_master_cpu_reserved: 250m
-# system_master_ephemeral_storage_reserved: 2Gi
+system_master_memory_reserved: 256Mi
+system_master_cpu_reserved: 250m
+system_master_ephemeral_storage_reserved: 2Gi
 
 ## Eviction Thresholds to avoid system OOMs
 # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#eviction-thresholds
@@ -331,32 +336,15 @@ persistent_volumes_enabled: false
 # nvidia_gpu_device_plugin_container: "registry.k8s.io/nvidia-gpu-device-plugin@sha256:0842734032018be107fa2490c98156992911e3e1f2a21e059ff0105b07dd8e9e"
 
 ## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
-# tls_min_version: ""
+tls_min_version: "VersionTLS12"
 
 ## Support tls cipher suites.
-# tls_cipher_suites: {}
-#   - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA
-#   - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
-#   - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
-#   - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
-#   - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
-#   - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256
-#   - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA
-#   - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA
-#   - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA
-#   - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
-#   - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
-#   - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA
-#   - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
-#   - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
-#   - TLS_ECDHE_RSA_WITH_RC4_128_SHA
-#   - TLS_RSA_WITH_3DES_EDE_CBC_SHA
-#   - TLS_RSA_WITH_AES_128_CBC_SHA
-#   - TLS_RSA_WITH_AES_128_CBC_SHA256
-#   - TLS_RSA_WITH_AES_128_GCM_SHA256
-#   - TLS_RSA_WITH_AES_256_CBC_SHA
-#   - TLS_RSA_WITH_AES_256_GCM_SHA384
-#   - TLS_RSA_WITH_RC4_128_SHA
+tls_cipher_suites:
+  - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+  - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256
+  - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+  - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
+  - TLS_RSA_WITH_AES_256_GCM_SHA384
 
 ## Amount of time to retain events. (default 1h0m0s)
 event_ttl_duration: "1h0m0s"

env/avroid_prod/inventory/group_vars/k8s_cluster/k8s-net-calico.yml

@@ -19,7 +19,7 @@ calico_cni_name: k8s-pod-network
 # add default ippool name
 # calico_pool_name: "default-pool"
 
-# add default ippool blockSize
+# add default ippool blockSize (defaults kube_network_node_prefix)
 calico_pool_blocksize: 26
 
 # add default ippool CIDR (must be inside kube_pods_subnet, defaults to kube_pods_subnet otherwise)

env/avroid_prod/inventory/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -1,271 +0,0 @@
----
-# cilium_version: "v1.15.4"
-
-# Log-level
-# cilium_debug: false
-
-# cilium_mtu: ""
-# cilium_enable_ipv4: true
-# cilium_enable_ipv6: false
-
-# Enable l2 announcement from cilium to replace Metallb Ref: https://docs.cilium.io/en/v1.14/network/l2-announcements/
-cilium_l2announcements: false
-
-# Cilium agent health port
-# cilium_agent_health_port: "9879"
-
-# Identity allocation mode selects how identities are shared between cilium
-# nodes by setting how they are stored. The options are "crd" or "kvstore".
-# - "crd" stores identities in kubernetes as CRDs (custom resource definition).
-#   These can be queried with:
-#     `kubectl get ciliumid`
-# - "kvstore" stores identities in an etcd kvstore.
-# - In order to support External Workloads, "crd" is required
-#   - Ref: https://docs.cilium.io/en/stable/gettingstarted/external-workloads/#setting-up-support-for-external-workloads-beta
-# - KVStore operations are only required when cilium-operator is running with any of the below options:
-#   - --synchronize-k8s-services
-#   - --synchronize-k8s-nodes
-#   - --identity-allocation-mode=kvstore
-#   - Ref: https://docs.cilium.io/en/stable/internals/cilium_operator/#kvstore-operations
-# cilium_identity_allocation_mode: kvstore
-
-# Etcd SSL dirs
-# cilium_cert_dir: /etc/cilium/certs
-# kube_etcd_cacert_file: ca.pem
-# kube_etcd_cert_file: cert.pem
-# kube_etcd_key_file: cert-key.pem
-
-# Limits for apps
-# cilium_memory_limit: 500M
-# cilium_cpu_limit: 500m
-# cilium_memory_requests: 64M
-# cilium_cpu_requests: 100m
-
-# Overlay Network Mode
-# cilium_tunnel_mode: vxlan
-
-# LoadBalancer Mode (snat/dsr/hybrid) Ref: https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#dsr-mode
-# cilium_loadbalancer_mode: snat
-
-# Optional features
-# cilium_enable_prometheus: false
-# Enable if you want to make use of hostPort mappings
-# cilium_enable_portmap: false
-# Monitor aggregation level (none/low/medium/maximum)
-# cilium_monitor_aggregation: medium
-# The monitor aggregation flags determine which TCP flags which, upon the
-# first observation, cause monitor notifications to be generated.
-#
-# Only effective when monitor aggregation is set to "medium" or higher.
-# cilium_monitor_aggregation_flags: "all"
-# Kube Proxy Replacement mode (strict/partial)
-# cilium_kube_proxy_replacement: partial
-
-# If upgrading from Cilium < 1.5, you may want to override some of these options
-# to prevent service disruptions. See also:
-# http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
-# cilium_preallocate_bpf_maps: false
-
-# `cilium_tofqdns_enable_poller` is deprecated in 1.8, removed in 1.9
-# cilium_tofqdns_enable_poller: false
-
-# `cilium_enable_legacy_services` is deprecated in 1.6, removed in 1.9
-# cilium_enable_legacy_services: false
-
-# Unique ID of the cluster. Must be unique across all connected clusters and
-# in the range of 1 and 255. Only relevant when building a mesh of clusters.
-# This value is not defined by default
-# cilium_cluster_id:
-
-# Deploy cilium even if kube_network_plugin is not cilium.
-# This enables to deploy cilium alongside another CNI to replace kube-proxy.
-# cilium_deploy_additionally: false
-
-# Auto direct nodes routes can be used to advertise pods routes in your cluster
-# without any tunneling (with `cilium_tunnel_mode` sets to `disabled`).
-# This works only if you have a L2 connectivity between all your nodes.
-# You wil also have to specify the variable `cilium_native_routing_cidr` to
-# make this work. Please refer to the cilium documentation for more
-# information about this kind of setups.
-# cilium_auto_direct_node_routes: false
-
-# Allows to explicitly specify the IPv4 CIDR for native routing.
-# When specified, Cilium assumes networking for this CIDR is preconfigured and
-# hands traffic destined for that range to the Linux network stack without
-# applying any SNAT.
-# Generally speaking, specifying a native routing CIDR implies that Cilium can
-# depend on the underlying networking stack to route packets to their
-# destination. To offer a concrete example, if Cilium is configured to use
-# direct routing and the Kubernetes CIDR is included in the native routing CIDR,
-# the user must configure the routes to reach pods, either manually or by
-# setting the auto-direct-node-routes flag.
-# cilium_native_routing_cidr: ""
-
-# Allows to explicitly specify the IPv6 CIDR for native routing.
-# cilium_native_routing_cidr_ipv6: ""
-
-# Enable transparent network encryption.
-# cilium_encryption_enabled: false
-
-# Encryption method. Can be either ipsec or wireguard.
-# Only effective when `cilium_encryption_enabled` is set to true.
-# cilium_encryption_type: "ipsec"
-
-# Enable encryption for pure node to node traffic.
-# This option is only effective when `cilium_encryption_type` is set to `ipsec`.
-# cilium_ipsec_node_encryption: false
-
-# If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation.
-# When this flag is enabled and Cilium detects that the kernel has no native support for WireGuard,
-# it will fallback on the wireguard-go user-space implementation of WireGuard.
-# This option is only effective when `cilium_encryption_type` is set to `wireguard`.
-# cilium_wireguard_userspace_fallback: false
-
-# IP Masquerade Agent
-# https://docs.cilium.io/en/stable/concepts/networking/masquerading/
-# By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded
-# cilium_ip_masq_agent_enable: false
-
-### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded
-# cilium_non_masquerade_cidrs:
-#   - 10.0.0.0/8
-#   - 172.16.0.0/12
-#   - 192.168.0.0/16
-#   - 100.64.0.0/10
-#   - 192.0.0.0/24
-#   - 192.0.2.0/24
-#   - 192.88.99.0/24
-#   - 198.18.0.0/15
-#   - 198.51.100.0/24
-#   - 203.0.113.0/24
-#   - 240.0.0.0/4
-### Indicates whether to masquerade traffic to the link local prefix.
-### If the masqLinkLocal is not set or set to false, then 169.254.0.0/16 is appended to the non-masquerade CIDRs list.
-# cilium_masq_link_local: false
-### A time interval at which the agent attempts to reload config from disk
-# cilium_ip_masq_resync_interval: 60s
-
-# Hubble
-### Enable Hubble without install
-# cilium_enable_hubble: false
-### Enable Hubble Metrics
-# cilium_enable_hubble_metrics: false
-### if cilium_enable_hubble_metrics: true
-# cilium_hubble_metrics: {}
-# - dns
-# - drop
-# - tcp
-# - flow
-# - icmp
-# - http
-### Enable Hubble install
-# cilium_hubble_install: false
-### Enable auto generate certs if cilium_hubble_install: true
-# cilium_hubble_tls_generate: false
-
-# IP address management mode for v1.9+.
-# https://docs.cilium.io/en/v1.9/concepts/networking/ipam/
-# cilium_ipam_mode: kubernetes
-
-# Extra arguments for the Cilium agent
-# cilium_agent_custom_args: []
-
-# For adding and mounting extra volumes to the cilium agent
-# cilium_agent_extra_volumes: []
-# cilium_agent_extra_volume_mounts: []
-
-# cilium_agent_extra_env_vars: []
-
-# cilium_operator_replicas: 2
-
-# The address at which the cillium operator bind health check api
-# cilium_operator_api_serve_addr: "127.0.0.1:9234"
-
-## A dictionary of extra config variables to add to cilium-config, formatted like:
-##  cilium_config_extra_vars:
-##    var1: "value1"
-##    var2: "value2"
-# cilium_config_extra_vars: {}
-
-# For adding and mounting extra volumes to the cilium operator
-# cilium_operator_extra_volumes: []
-# cilium_operator_extra_volume_mounts: []
-
-# Extra arguments for the Cilium Operator
-# cilium_operator_custom_args: []
-
-# Name of the cluster. Only relevant when building a mesh of clusters.
-# cilium_cluster_name: default
-
-# Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`.
-# This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime.
-# Available for Cilium v1.10 and up.
-# cilium_cni_exclusive: true
-
-# Configure the log file for CNI logging with retention policy of 7 days.
-# Disable CNI file logging by setting this field to empty explicitly.
-# Available for Cilium v1.12 and up.
-# cilium_cni_log_file: "/var/run/cilium/cilium-cni.log"
-
-# -- Configure cgroup related configuration
-# -- Enable auto mount of cgroup2 filesystem.
-# When `cilium_cgroup_auto_mount` is enabled, cgroup2 filesystem is mounted at
-# `cilium_cgroup_host_root` path on the underlying host and inside the cilium agent pod.
-# If users disable `cilium_cgroup_auto_mount`, it's expected that users have mounted
-# cgroup2 filesystem at the specified `cilium_cgroup_auto_mount` volume, and then the
-# volume will be mounted inside the cilium agent pod at the same path.
-# Available for Cilium v1.11 and up
-# cilium_cgroup_auto_mount: true
-# -- Configure cgroup root where cgroup2 filesystem is mounted on the host
-# cilium_cgroup_host_root: "/run/cilium/cgroupv2"
-
-# Specifies the ratio (0.0-1.0) of total system memory to use for dynamic
-# sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
-# cilium_bpf_map_dynamic_size_ratio: "0.0"
-
-# -- Enables masquerading of IPv4 traffic leaving the node from endpoints.
-# Available for Cilium v1.10 and up
-# cilium_enable_ipv4_masquerade: true
-# -- Enables masquerading of IPv6 traffic leaving the node from endpoints.
-# Available for Cilium v1.10 and up
-# cilium_enable_ipv6_masquerade: true
-
-# -- Enable native IP masquerade support in eBPF
-# cilium_enable_bpf_masquerade: false
-
-# -- Configure whether direct routing mode should route traffic via
-# host stack (true) or directly and more efficiently out of BPF (false) if
-# the kernel supports it. The latter has the implication that it will also
-# bypass netfilter in the host namespace.
-# cilium_enable_host_legacy_routing: true
-
-# -- Enable use of the remote node identity.
-# ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity
-# cilium_enable_remote_node_identity: true
-
-# -- Enable the use of well-known identities.
-# cilium_enable_well_known_identities: false
-
-# cilium_enable_bpf_clock_probe: true
-
-# -- Whether to enable CNP status updates.
-# cilium_disable_cnp_status_updates: true
-
-# A list of extra rules variables to add to clusterrole for cilium operator, formatted like:
-#   cilium_clusterrole_rules_operator_extra_vars:
-#     - apiGroups:
-#       - '""'
-#       resources:
-#       - pods
-#       verbs:
-#       - delete
-#     - apiGroups:
-#       - '""'
-#       resources:
-#       - nodes
-#       verbs:
-#       - list
-#       - watch
-#       resourceNames:
-#       - toto
-# cilium_clusterrole_rules_operator_extra_vars: []

env/avroid_prod/inventory/group_vars/k8s_cluster/k8s-net-custom-cni.yml

@@ -1,51 +0,0 @@
----
-# custom_cni network plugin configuration
-# There are two deployment options to choose from, select one
-
-## OPTION 1 - Static manifest files
-## With this option, referred manifest file will be deployed
-## as if the `kubectl apply -f` method was used with it.
-#
-## List of Kubernetes resource manifest files
-## See tests/files/custom_cni/README.md for example
-# custom_cni_manifests: []
-
-## OPTION 1 EXAMPLE - Cilium static manifests in Kubespray tree
-# custom_cni_manifests:
-#   - "{{ playbook_dir }}/../tests/files/custom_cni/cilium.yaml"
-
-## OPTION 2 - Helm chart application
-## This allows the CNI backend to be deployed to Kubespray cluster
-## as common Helm application.
-#
-## Helm release name - how the local instance of deployed chart will be named
-# custom_cni_chart_release_name: ""
-#
-## Kubernetes namespace to deploy into
-# custom_cni_chart_namespace: "kube-system"
-#
-## Helm repository name - how the local record of Helm repository will be named
-# custom_cni_chart_repository_name: ""
-#
-## Helm repository URL
-# custom_cni_chart_repository_url: ""
-#
-## Helm chart reference - path to the chart in the repository
-# custom_cni_chart_ref: ""
-#
-## Helm chart version
-# custom_cni_chart_version: ""
-#
-## Custom Helm values to be used for deployment
-# custom_cni_chart_values: {}
-
-## OPTION 2 EXAMPLE - Cilium deployed from official public Helm chart
-# custom_cni_chart_namespace: kube-system
-# custom_cni_chart_release_name: cilium
-# custom_cni_chart_repository_name: cilium
-# custom_cni_chart_repository_url: https://helm.cilium.io
-# custom_cni_chart_ref: cilium/cilium
-# custom_cni_chart_version: 1.14.3
-# custom_cni_chart_values:
-#   cluster:
-#     name: "cilium-demo"

env/avroid_prod/inventory/group_vars/k8s_cluster/k8s-net-flannel.yml

@@ -1,18 +0,0 @@
-# see roles/network_plugin/flannel/defaults/main.yml
-
-## interface that should be used for flannel operations
-## This is actually an inventory cluster-level item
-# flannel_interface:
-
-## Select interface that should be used for flannel operations by regexp on Name or IP
-## This is actually an inventory cluster-level item
-## example: select interface with ip from net 10.0.0.0/23
-## single quote and escape backslashes
-# flannel_interface_regexp: '10\\.0\\.[0-2]\\.\\d{1,3}'
-
-# You can choose what type of flannel backend to use: 'vxlan',  'host-gw' or 'wireguard'
-# please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md
-# flannel_backend_type: "vxlan"
-# flannel_vxlan_vni: 1
-# flannel_vxlan_port: 8472
-# flannel_vxlan_direct_routing: false

env/avroid_prod/inventory/group_vars/k8s_cluster/k8s-net-kube-ovn.yml

@@ -1,63 +0,0 @@
----
-
-# geneve or vlan
-kube_ovn_network_type: geneve
-
-# geneve, vxlan or stt. ATTENTION: some networkpolicy cannot take effect when using vxlan and stt need custom compile ovs kernel module
-kube_ovn_tunnel_type: geneve
-
-## The nic to support container network can be a nic name or a group of regex separated by comma e.g: 'enp6s0f0,eth.*', if empty will use the nic that the default route use.
-# kube_ovn_iface: eth1
-## The MTU used by pod iface in overlay networks (default iface MTU - 100)
-# kube_ovn_mtu: 1333
-
-## Enable hw-offload, disable traffic mirror and set the iface to the physical port. Make sure that there is an IP address bind to the physical port.
-kube_ovn_hw_offload: false
-# traffic mirror
-kube_ovn_traffic_mirror: false
-
-# kube_ovn_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112
-# kube_ovn_default_interface_name: eth0
-
-kube_ovn_external_address: 8.8.8.8
-kube_ovn_external_address_ipv6: 2400:3200::1
-kube_ovn_external_dns: alauda.cn
-
-# kube_ovn_default_gateway: 10.233.64.1,fd85:ee78:d8a6:8607::1:0
-kube_ovn_default_gateway_check: true
-kube_ovn_default_logical_gateway: false
-# kube_ovn_default_exclude_ips: 10.16.0.1
-kube_ovn_node_switch_cidr: 100.64.0.0/16
-kube_ovn_node_switch_cidr_ipv6: fd00:100:64::/64
-
-## vlan config, set default interface name and vlan id
-# kube_ovn_default_interface_name: eth0
-kube_ovn_default_vlan_id: 100
-kube_ovn_vlan_name: product
-
-## pod nic type, support: veth-pair or internal-port
-kube_ovn_pod_nic_type: veth_pair
-
-## Enable load balancer
-kube_ovn_enable_lb: true
-
-## Enable network policy support
-kube_ovn_enable_np: true
-
-## Enable external vpc support
-kube_ovn_enable_external_vpc: true
-
-## Enable checksum
-kube_ovn_encap_checksum: true
-
-## enable ssl
-kube_ovn_enable_ssl: false
-
-## dpdk
-kube_ovn_dpdk_enabled: false
-
-## enable interconnection to an existing IC database server.
-kube_ovn_ic_enable: false
-kube_ovn_ic_autoroute: true
-kube_ovn_ic_dbhost: "127.0.0.1"
-kube_ovn_ic_zone: "kubernetes"

env/avroid_prod/inventory/group_vars/k8s_cluster/k8s-net-kube-router.yml

@@ -1,73 +0,0 @@
-# See roles/network_plugin/kube-router/defaults/main.yml
-
-# Kube router version
-# Default to v2
-# kube_router_version: "v2.0.0"
-# Uncomment to use v1 (Deprecated)
-# kube_router_version: "v1.6.0"
-
-# Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP
-# kube_router_run_router: true
-
-# Enables Network Policy -- sets up iptables to provide ingress firewall for pods
-# kube_router_run_firewall: true
-
-# Enables Service Proxy -- sets up IPVS for Kubernetes Services
-# see docs/kube-router.md "Caveats" section
-# kube_router_run_service_proxy: false
-
-# Add Cluster IP of the service to the RIB so that it gets advertises to the BGP peers.
-# kube_router_advertise_cluster_ip: false
-
-# Add External IP of service to the RIB so that it gets advertised to the BGP peers.
-# kube_router_advertise_external_ip: false
-
-# Add LoadBalancer IP of service status as set by the LB provider to the RIB so that it gets advertised to the BGP peers.
-# kube_router_advertise_loadbalancer_ip: false
-
-# Enables BGP graceful restarts
-# kube_router_bgp_graceful_restart: true
-
-# Adjust manifest of kube-router daemonset template with DSR needed changes
-# kube_router_enable_dsr: false
-
-# Array of arbitrary extra arguments to kube-router, see
-# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md
-# kube_router_extra_args: []
-
-# ASN number of the cluster, used when communicating with external BGP routers
-# kube_router_cluster_asn: ~
-
-# ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr.
-# kube_router_peer_router_asns: ~
-
-# The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's.
-# kube_router_peer_router_ips: ~
-
-# The remote port of the external BGP to which all nodes will peer. If not set, default BGP port (179) will be used.
-# kube_router_peer_router_ports: ~
-
-# Setups node CNI to allow hairpin mode, requires node reboots, see
-# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode
-# kube_router_support_hairpin_mode: false
-
-# Select DNS Policy ClusterFirstWithHostNet, ClusterFirst, etc.
-# kube_router_dns_policy: ClusterFirstWithHostNet
-
-# Array of annotations for master
-# kube_router_annotations_master: []
-
-# Array of annotations for every node
-# kube_router_annotations_node: []
-
-# Array of common annotations for every node
-# kube_router_annotations_all: []
-
-# Enables scraping kube-router metrics with Prometheus
-# kube_router_enable_metrics: false
-
-# Path to serve Prometheus metrics on
-# kube_router_metrics_path: /metrics
-
-# Prometheus metrics port to use
-# kube_router_metrics_port: 9255

env/avroid_prod/inventory/group_vars/k8s_cluster/k8s-net-macvlan.yml

@@ -1,6 +0,0 @@
----
-# private interface, on a l2-network
-macvlan_interface: "eth1"
-
-# Enable nat in default gateway network interface
-enable_nat_default_gateway: true

env/avroid_prod/inventory/group_vars/k8s_cluster/k8s-net-weave.yml

@@ -1,64 +0,0 @@
-# see roles/network_plugin/weave/defaults/main.yml
-
-# Weave's network password for encryption, if null then no network encryption.
-# weave_password: ~
-
-# If set to 1, disable checking for new Weave Net versions (default is blank,
-# i.e. check is enabled)
-# weave_checkpoint_disable: false
-
-# Soft limit on the number of connections between peers. Defaults to 100.
-# weave_conn_limit: 100
-
-# Weave Net defaults to enabling hairpin on the bridge side of the veth pair
-# for containers attached. If you need to disable hairpin, e.g. your kernel is
-# one of those that can panic if hairpin is enabled, then you can disable it by
-# setting `HAIRPIN_MODE=false`.
-# weave_hairpin_mode: true
-
-# The range of IP addresses used by Weave Net and the subnet they are placed in
-# (CIDR format; default 10.32.0.0/12)
-# weave_ipalloc_range: "{{ kube_pods_subnet }}"
-
-# Set to 0 to disable Network Policy Controller (default is on)
-# weave_expect_npc: "{{ enable_network_policy }}"
-
-# List of addresses of peers in the Kubernetes cluster (default is to fetch the
-# list from the api-server)
-# weave_kube_peers: ~
-
-# Set the initialization mode of the IP Address Manager (defaults to consensus
-# amongst the KUBE_PEERS)
-# weave_ipalloc_init: ~
-
-# Set the IP address used as a gateway from the Weave network to the host
-# network - this is useful if you are configuring the addon as a static pod.
-# weave_expose_ip: ~
-
-# Address and port that the Weave Net daemon will serve Prometheus-style
-# metrics on (defaults to 0.0.0.0:6782)
-# weave_metrics_addr: ~
-
-# Address and port that the Weave Net daemon will serve status requests on
-# (defaults to disabled)
-# weave_status_addr: ~
-
-# Weave Net defaults to 1376 bytes, but you can set a smaller size if your
-# underlying network has a tighter limit, or set a larger size for better
-# performance if your network supports jumbo frames (e.g. 8916)
-# weave_mtu: 1376
-
-# Set to 1 to preserve the client source IP address when accessing Service
-# annotated with `service.spec.externalTrafficPolicy=Local`. The feature works
-# only with Weave IPAM (default).
-# weave_no_masq_local: true
-
-# set to nft to use nftables backend for iptables (default is iptables)
-# weave_iptables_backend: iptables
-
-# Extra variables that passing to launch.sh, useful for enabling seed mode, see
-# https://www.weave.works/docs/net/latest/tasks/ipam/ipam/
-# weave_extra_args: ~
-
-# Extra variables for weave_npc that passing to launch.sh, useful for change log level, ex --log-level=error
-# weave_npc_extra_args: ~