diff --git a/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/README.md b/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/README.md new file mode 100644 index 0000000..bc19001 --- /dev/null +++ b/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/README.md @@ -0,0 +1,62 @@ +# Install [Prometheus](https://github.com/prometheus-community/helm-charts/blob/main/charts/prometheus/README.md) + +## Create from version +| Project | Version | +|------------|----------| +| Prometheus | 27.5.1 | + +## Install +Настраиваем для работы файл конфигурации kubectl для подключения к кластеру Kubernetes + +Готовим служебную УЗ для vault (ЭТО АВТОМАТИЗИРОВАНО через ArgoCD): +```bash +kubectl apply -f .rbac/vault-service-account.yaml +kubectl apply -f .rbac/harbor-registry-secret.yaml +``` + +Получаем из Vault секреты с паролями/токенами/сертификатами и выгружаем в окружение оболочки: +```bash +./prometheus_values_secrets_init.sh + +source .creds +``` + +Применяем сетевые политики: +```bash +kubectl -n cloud-prod apply -f prometheus-network-policy.yaml +``` + +И производим непосредственную установку: + +```bash +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update + +helm install -n prometheus prometheus-community/prometheus -f values-override.yaml \ + --set serverFiles.prometheus.yml.scrape_configs[1].consul_sd_configs[0].authorization.credentials=$PROMETHEUS_CONSUL_TOKEN_VALUE \ + --set serverFiles.prometheus.yml.scrape_configs[2].consul_sd_configs[0].authorization.credentials=$PROMETHEUS_CONSUL_TOKEN_VALUE \ + --set serverFiles.prometheus.yml.scrape_configs[3].consul_sd_configs[0].authorization.credentials=$PROMETHEUS_CONSUL_TOKEN_VALUE +``` + +## Upgrade/Changes +Обновление настроек и чарта делаем так: + +В начале сравниваем содержимое `values-override.yaml` с исходным файлом `values.yaml` +(ссылка на исходную версию зафиксирована в заголовке `values-override.yaml`) через diff (IDE лучше). +Или с новой версией `values.yaml` в Helm-чарте. +Вносим необходимые правки, новые опции или добавляем секреты. + +Получаем из Vault секреты с паролями/токенами/сертификатами и выгружаем в окружение оболочки: +```bash +./prometheus_values_secrets_init.sh + +source .creds +``` + +Затем применяем: +```bash +helm upgrade -n prometheus prometheus-community/prometheus -f values-override.yaml \ + --set serverFiles.prometheus.yml.scrape_configs[1].consul_sd_configs[0].authorization.credentials=$PROMETHEUS_CONSUL_TOKEN_VALUE \ + --set serverFiles.prometheus.yml.scrape_configs[2].consul_sd_configs[0].authorization.credentials=$PROMETHEUS_CONSUL_TOKEN_VALUE \ + --set serverFiles.prometheus.yml.scrape_configs[3].consul_sd_configs[0].authorization.credentials=$PROMETHEUS_CONSUL_TOKEN_VALUE +``` diff --git a/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/argocd-apps-prometheus-app.yaml b/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/argocd-apps-prometheus-app.yaml deleted file mode 100644 index 3f544bd..0000000 --- a/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/argocd-apps-prometheus-app.yaml +++ /dev/null @@ -1,51 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: cloud-prometheus - namespace: argocd - finalizers: - - resources-finalizer.argocd.argoproj.io -spec: - project: cloud-prometheus - destination: - server: https://kubernetes.default.svc - namespace: cloud-prod - sources: - - repoURL: https://git.avroid.tech/K8s/k8s-configs.git - targetRevision: master - ref: values - - repoURL: https://nexus.avroid.tech/repository/devops-helm-proxy-helm/ - chart: "prometheus-community/prometheus" - targetRevision: 27.5.1 - helm: - valueFiles: - - $values/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/values-override.yaml - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - ApplyOutOfSyncOnly=true - - CreateNamespace=true ---- -apiVersion: argoproj.io/v1alpha1 -kind: AppProject -metadata: - name: cloud-prometheus - namespace: argocd - # Finalizer that ensures that project is not deleted until it is not referenced by any application - finalizers: - - resources-finalizer.argocd.argoproj.io -spec: - sourceRepos: - - https://git.avroid.tech/K8s/k8s-configs.git - - https://nexus.avroid.tech/repository/devops-helm-proxy-helm/ - # Only permit applications to deploy to the guestbook namespace in the same cluster - destinations: - - namespace: cloud-prod - server: https://kubernetes.default.svc - # Deny all cluster-scoped resources from being created, except for Namespace - clusterResourceWhitelist: - - group: '' - kind: Namespace diff --git a/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/argocd-apps-prometheus-secret.yaml b/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/argocd-apps-prometheus-secret.yaml deleted file mode 100644 index b8fa314..0000000 --- a/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/argocd-apps-prometheus-secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - app.kubernetes.io/managed-by: argocd - name: prometheus-secret - namespace: cloud-prod - annotations: - avp.kubernetes.io/path: "team-devops/data/services/monitoring/k8s/cloud-prod/prometheus" -type: Opaque -stringData: - consul_secret: diff --git a/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/argocd-apps-prometheus-network-policy.yaml b/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/prometheus-network-policy.yaml similarity index 100% rename from clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/argocd-apps-prometheus-network-policy.yaml rename to clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/prometheus-network-policy.yaml diff --git a/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/prometheus_values_secrets_init.sh b/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/prometheus_values_secrets_init.sh new file mode 100755 index 0000000..7ebea8c --- /dev/null +++ b/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/prometheus_values_secrets_init.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -e + +rm -rf .creds + +prometheus_consul_token_value=$(vault kv get team-devops//services/monitoring/k8s/cloud-prod/prometheus | grep consul.secret | awk '{print $2}') + +cat > .creds << EOF +export PROMETHEUS_CONSUL_TOKEN_VALUE='${prometheus_consul_token_value}' +EOF + +echo "Run:" +echo ' 1. source .creds' +echo ' 2. kubectl -n cloud-prod apply -f prometheus-network-policy.yaml' diff --git a/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/values-override.yaml b/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/values-override.yaml index 4502942..59bf374 100644 --- a/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/values-override.yaml +++ b/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/values-override.yaml @@ -138,7 +138,7 @@ serverFiles: tags: [ k8s_postgres_exporter ] services: [ monitoring_k8s_postgres_exporter ] authorization: - credentials_file: /etc/secrets/consul_secret + credentials: consul_secret relabel_configs: - source_labels: [ __meta_consul_service_metadata_metrics_path ] target_label: __metrics_path__ @@ -165,7 +165,7 @@ serverFiles: tags: [ k8s_redis_exporter ] services: [ monitoring_k8s_redis_exporter ] authorization: - credentials: /etc/secrets/consul_secret + credentials: consul_secret relabel_configs: - source_labels: [ __meta_consul_service_metadata_metrics_path ] target_label: __metrics_path__ @@ -187,7 +187,7 @@ serverFiles: tags: [ k8s_patroni_exporter ] services: [ monitoring_k8s_patroni_exporter ] authorization: - credentials: /etc/secrets/consul_secret + credentials: consul_secret relabel_configs: - source_labels: [ __meta_consul_service_metadata_metrics_path ] target_label: __metrics_path__