[DO-1477] create k8s struct and move openresty (#2)
Co-authored-by: Rustam Tagaev <rustam.tagaev@avroid.tech> Co-authored-by: Denis Patrakeev <denis.patrakeev@avroid.team> Reviewed-on: https://git.avroid.tech/K8s/k8s-configs/pulls/2 Reviewed-by: Denis Patrakeev <denis.patrakeev@avroid.team> Co-authored-by: Rustam Tagaev <rustam.tagaev@avroid.team> Co-committed-by: Rustam Tagaev <rustam.tagaev@avroid.team>
This commit is contained in:
Rustam Tagaev
committed by
Denis Patrakeev
Denis Patrakeev
parent
9962ddb2bc
commit
90155cad0b
94
README.md
94
README.md
@@ -1,2 +1,96 @@
|
|||||||
# k8s-configs
|
# k8s-configs
|
||||||
|
|
||||||
|
## Настройка внешних секретов
|
||||||
|
|
||||||
|
[Ссылка на офф. доку](https://bank-vaults.dev)
|
||||||
|
|
||||||
|
Для создания vault injector нужно установить helm
|
||||||
|
|
||||||
|
```bash
|
||||||
|
helm upgrade -n vault-infra --install --wait vault-secrets-webhook oci://ghcr.io/bank-vaults/helm-charts/vault-secrets-webhook
|
||||||
|
```
|
||||||
|
|
||||||
|
Далее создать роль секрет и рольбиндинг
|
||||||
|
|
||||||
|
```bash
|
||||||
|
kubectl apply -f clusters/k8s-avroid-office.prod.local/namespaces/vault-infra/vault-secrets-webhook/vault-service-account.yaml
|
||||||
|
kubectl apply -f clusters/k8s-avroid-office.prod.local/namespaces/vault-infra/vault-secrets-webhook/vault-cluster_role_binding.yaml
|
||||||
|
kubectl apply -f clusters/k8s-avroid-office.prod.local/namespaces/vault-infra/vault-secrets-webhook/vault-secret.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
Настройка со стороны vault
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vault auth enable -path=avroid-office kubernetes
|
||||||
|
|
||||||
|
TOKEN=$(kubectl get secret vault -n vault-infra -o jsonpath="{.data.token}" | base64 --decode)
|
||||||
|
CA_CERT=$(kubectl get secret vault -n vault-infra -o jsonpath="{.data['ca\.crt']}" | base64 --decode)
|
||||||
|
ISSUER=$(kubectl get --raw /.well-known/openid-configuration | jq '.issuer')
|
||||||
|
K8S_CLUSTER=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')
|
||||||
|
|
||||||
|
vault write auth/avroid-office/config \
|
||||||
|
kubernetes_host="${K8S_CLUSTER}" \
|
||||||
|
token_reviewer_jwt="${TOKEN}" \
|
||||||
|
kubernetes_ca_cert="${CA_CERT}" \
|
||||||
|
issuer="${ISSUER}" \
|
||||||
|
disable_local_ca_jwt="true"
|
||||||
|
```
|
||||||
|
|
||||||
|
Далее создаем app роль - на каждый namespace нужно создавать свою роль с одноименным названием.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
vault write auth/avroid-office/role/tavro-cloud-dev \
|
||||||
|
bound_service_account_names="*" \
|
||||||
|
bound_service_account_namespaces="tavro-cloud-dev" \
|
||||||
|
policies="prj-tavro-cloud-backend" \
|
||||||
|
ttl="24h"
|
||||||
|
```
|
||||||
|
|
||||||
|
policies - содержит список vault полиси, если нужно добавить новый, то просто добавляем и выполняем эту команду
|
||||||
|
|
||||||
|
ВНИМАНИЕ: для нормальной работы должен быть создан service-account в каждом namespace
|
||||||
|
|
||||||
|
для этого просто нужно выполнить
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# не забудь поменять в файле namespace свой
|
||||||
|
cp clusters/k8s-avroid-office.prod.local/namespaces/vault-infra/vault-secrets-webhook/vault-service-account.yaml ./<your_namespace>
|
||||||
|
# далее меняем в файле имя на свой namespace и запускаем
|
||||||
|
kubectl apply -f vault-service-account.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
Простой пример для тестирования - в логах вы увидите свой секрет
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
kubectl apply -n sandbox -f - <<"EOF"
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: vault-test
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: vault
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: vault
|
||||||
|
annotations:
|
||||||
|
vault.security.banzaicloud.io/vault-addr: "https://vault.avroid.tech" # внешний адрес vault
|
||||||
|
vault.security.banzaicloud.io/vault-role: "vault-k8s-role" # роль из под которой будем ходить в vault
|
||||||
|
vault.security.banzaicloud.io/vault-skip-verify: "false" # проверять сертификат или нет на стороне vault
|
||||||
|
# vault.security.banzaicloud.io/vault-tls-secret: "vault-tls" # сертификат для vault если он самоподписанный
|
||||||
|
# vault.security.banzaicloud.io/vault-agent: "false" # запускать акента который будет отслеживать изменения секрета
|
||||||
|
vault.security.banzaicloud.io/vault-path: "avroid-office" # название kubernetes аутентификации в vault
|
||||||
|
spec:
|
||||||
|
serviceAccountName: vault # имя сервиса аккаунта - должен быть в каждом namespace
|
||||||
|
containers:
|
||||||
|
- name: alpine
|
||||||
|
image: alpine
|
||||||
|
command: ["sh", "-c", "echo $POSTGRES_DSN && echo going to sleep... && sleep 10000"]
|
||||||
|
env:
|
||||||
|
- name: POSTGRES_DSN # переменная окружения куда попадет секрет
|
||||||
|
value: vault:prj-tavro-cloud-backend/data/k8s/avroid.local/ns-tarvo-cloud-dev/svc-messenger-core-api#POSTGRES_DSN # путь до секрета
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/managed-by: manual
|
||||||
|
name: harbor-registry-secret
|
||||||
|
namespace: jenkins-builds
|
||||||
|
type: kubernetes.io/dockerconfigjson
|
||||||
|
data:
|
||||||
|
.dockerconfigjson: eyJhdXRocyI6eyJoYXJib3IuYXZyb2lkLnRlY2giOnsidXNlcm5hbWUiOiJyb2JvdCRjaSIsInBhc3N3b3JkIjoiSFJqOWlIQXh2VUl1eVRab2d1S1BkR21US082UjlkUnoiLCJhdXRoIjoiY205aWIzUWtZMms2U0ZKcU9XbElRWGgyVlVsMWVWUmFiMmQxUzFCa1IyMVVTMDgyVWpsa1Vubz0ifX19
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/managed-by: manual
|
||||||
|
name: jenkins
|
||||||
|
namespace: jenkins-builds
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: Role
|
||||||
|
name: jenkins
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: jenkins
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/managed-by: manual
|
||||||
|
name: jenkins
|
||||||
|
namespace: jenkins-builds
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- pods
|
||||||
|
- persistentvolumes
|
||||||
|
- persistentvolumeclaims
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- pods/exec
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- delete
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- pods/log
|
||||||
|
- storageclass
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- events
|
||||||
|
- nodes
|
||||||
|
verbs:
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- secrets
|
||||||
|
- nodes
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: jenkins
|
||||||
|
namespace: jenkins-builds
|
||||||
|
annotations:
|
||||||
|
kubernetes.io/service-account.name: jenkins
|
||||||
|
labels:
|
||||||
|
name: jenkins-builds
|
||||||
|
app.kubernetes.io/managed-by: manual
|
||||||
|
type: kubernetes.io/service-account-token
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
name: jenkins
|
||||||
|
app.kubernetes.io/managed-by: manual
|
||||||
|
name: jenkins
|
||||||
|
namespace: jenkins-builds
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: jenkins-builds
|
||||||
|
labels:
|
||||||
|
name: jenkins-builds
|
||||||
|
app.kubernetes.io/managed-by: manual
|
||||||
|
annotations:
|
||||||
|
scheduler.alpha.kubernetes.io/node-selector: node-role.kubernetes.io/build=
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/managed-by: manual
|
||||||
|
name: jenkins-deploy
|
||||||
|
namespace: jenkins-builds
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: harbor-registry-secret
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: serviceaccount
|
||||||
|
app.kubernetes.io/instance: vault-sa
|
||||||
|
app.kubernetes.io/component: rbac
|
||||||
|
app.kubernetes.io/created-by: vault-operator
|
||||||
|
app.kubernetes.io/part-of: vault-operator
|
||||||
|
app.kubernetes.io/managed-by: kustomize
|
||||||
|
name: vault
|
||||||
|
namespace: sandbox
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: sandbox
|
||||||
|
labels:
|
||||||
|
name: sandbox
|
||||||
|
app.kubernetes.io/managed-by: manual
|
||||||
|
annotations:
|
||||||
|
scheduler.alpha.kubernetes.io/node-selector: node-role.kubernetes.io/worker=
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ResourceQuota
|
||||||
|
metadata:
|
||||||
|
name: sandbox
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/managed-by: manual
|
||||||
|
spec:
|
||||||
|
hard:
|
||||||
|
requests.cpu: "8"
|
||||||
|
requests.memory: 24Gi
|
||||||
|
limits.cpu: "16"
|
||||||
|
limits.memory: 32Gi
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: avroid-tech-tls
|
||||||
|
namespace: tavro-cloud-dev
|
||||||
|
data:
|
||||||
|
# base64 encoded cert see values in vault. Don't push it to git!
|
||||||
|
tls.crt: ""
|
||||||
|
tls.key: ""
|
||||||
|
type: kubernetes.io/tls
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/managed-by: manual
|
||||||
|
name: jenkins-deploy-tavro-cloud-dev
|
||||||
|
namespace: tavro-cloud-dev
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: Role
|
||||||
|
name: tavro-cloud-dev-full
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: jenkins-deploy
|
||||||
|
namespace: jenkins-builds
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/managed-by: manual
|
||||||
|
name: harbor-registry-secret
|
||||||
|
namespace: tavro-cloud-dev
|
||||||
|
type: kubernetes.io/dockerconfigjson
|
||||||
|
data:
|
||||||
|
.dockerconfigjson: eyJhdXRocyI6eyJoYXJib3IuYXZyb2lkLnRlY2giOnsidXNlcm5hbWUiOiJyb2JvdCRjaSIsInBhc3N3b3JkIjoiSFJqOWlIQXh2VUl1eVRab2d1S1BkR21US082UjlkUnoiLCJhdXRoIjoiY205aWIzUWtZMms2U0ZKcU9XbElRWGgyVlVsMWVWUmFiMmQxUzFCa1IyMVVTMDgyVWpsa1Vubz0ifX19
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
name: tavro-cloud-dev-full
|
||||||
|
namespace: tavro-cloud-dev
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/managed-by: manual
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- '*'
|
||||||
|
resources:
|
||||||
|
- '*'
|
||||||
|
verbs:
|
||||||
|
- '*'
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: serviceaccount
|
||||||
|
app.kubernetes.io/instance: vault-sa
|
||||||
|
app.kubernetes.io/component: rbac
|
||||||
|
app.kubernetes.io/created-by: vault-operator
|
||||||
|
app.kubernetes.io/part-of: vault-operator
|
||||||
|
app.kubernetes.io/managed-by: kustomize
|
||||||
|
name: vault
|
||||||
|
namespace: tavro-cloud-dev
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
# Install
|
||||||
|
|
||||||
|
Для установки нужно забрать values и использовать их. Тут указан только пример, так как все это делается через jenkins
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -o values.yaml https://git.avroid.tech/Apps-Backend/cloud-messenger-core-api/src/branch/develop/.helm/values.dev.yaml
|
||||||
|
helm upgrade --install -f values.yaml cloud-messenger-core-api avroid/cloud-messenger-core-api
|
||||||
|
```
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
# see https://git.avroid.tech/Apps-Backend/helm-values/src/branch/master/avroid.local/api-gateway/openresty/values.yaml
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: tavro-cloud-dev
|
||||||
|
labels:
|
||||||
|
name: tavro-cloud-dev
|
||||||
|
app.kubernetes.io/managed-by: manual
|
||||||
|
annotations:
|
||||||
|
scheduler.alpha.kubernetes.io/node-selector: node-role.kubernetes.io/worker=
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ResourceQuota
|
||||||
|
metadata:
|
||||||
|
name: tavro-cloud-dev
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/managed-by: manual
|
||||||
|
spec:
|
||||||
|
hard:
|
||||||
|
requests.cpu: "8"
|
||||||
|
requests.memory: 24Gi
|
||||||
|
limits.cpu: "16"
|
||||||
|
limits.memory: 32Gi
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: vault-infra
|
||||||
|
labels:
|
||||||
|
name: vault-infra
|
||||||
|
app.kubernetes.io/managed-by: manual
|
||||||
|
annotations:
|
||||||
|
scheduler.alpha.kubernetes.io/node-selector: node-role.kubernetes.io/worker=
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
# helm upgrade -n vault-infra --install --wait vault-secrets-webhook oci://ghcr.io/bank-vaults/helm-charts/vault-secrets-webhook
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: clusterrolebinding
|
||||||
|
app.kubernetes.io/instance: manager-rolebinding
|
||||||
|
app.kubernetes.io/component: rbac
|
||||||
|
app.kubernetes.io/created-by: vault-operator
|
||||||
|
app.kubernetes.io/part-of: vault-operator
|
||||||
|
app.kubernetes.io/managed-by: kustomize
|
||||||
|
name: vault-auth-delegator
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole
|
||||||
|
name: system:auth-delegator
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: vault
|
||||||
|
namespace: vault-infra
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: vault
|
||||||
|
namespace: vault-infra
|
||||||
|
annotations:
|
||||||
|
kubernetes.io/service-account.name: vault
|
||||||
|
type: kubernetes.io/service-account-token
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: serviceaccount
|
||||||
|
app.kubernetes.io/instance: vault-sa
|
||||||
|
app.kubernetes.io/component: rbac
|
||||||
|
app.kubernetes.io/created-by: vault-operator
|
||||||
|
app.kubernetes.io/part-of: vault-operator
|
||||||
|
app.kubernetes.io/managed-by: kustomize
|
||||||
|
name: vault
|
||||||
|
namespace: vault-infra
|
||||||
Reference in New Issue
Block a user