[hotfix] Move prometheus to manual (!104)

Co-authored-by: denis.patrakeev <denis.patrakeev@avroid.tech> Reviewed-on: https://git.avroid.tech/K8s/k8s-configs/pulls/104
2025-03-04 21:19:51 +03:00
parent 280a76adb6
commit 072c3e09ed
6 changed files with 79 additions and 67 deletions
--- a/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/README.md
+++ b/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/README.md
@@ -0,0 +1,62 @@
+# Install [Prometheus](https://github.com/prometheus-community/helm-charts/blob/main/charts/prometheus/README.md)
+
+## Create from version
+| Project    | Version  |
+|------------|----------|
+| Prometheus | 27.5.1   |
+
+## Install
+Настраиваем для работы файл конфигурации kubectl для подключения к кластеру Kubernetes
+
+Готовим служебную УЗ для vault (ЭТО АВТОМАТИЗИРОВАНО через ArgoCD):
+```bash
+kubectl apply -f .rbac/vault-service-account.yaml
+kubectl apply -f .rbac/harbor-registry-secret.yaml
+```
+
+Получаем из Vault секреты с паролями/токенами/сертификатами и выгружаем в окружение оболочки:
+```bash
+./prometheus_values_secrets_init.sh
+
+source .creds
+```
+
+Применяем сетевые политики:
+```bash
+kubectl -n cloud-prod apply -f prometheus-network-policy.yaml
+```
+
+И производим непосредственную установку:
+
+```bash
+helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+helm repo update
+
+helm install -n prometheus prometheus-community/prometheus -f values-override.yaml \
+  --set serverFiles.prometheus.yml.scrape_configs[1].consul_sd_configs[0].authorization.credentials=$PROMETHEUS_CONSUL_TOKEN_VALUE \
+  --set serverFiles.prometheus.yml.scrape_configs[2].consul_sd_configs[0].authorization.credentials=$PROMETHEUS_CONSUL_TOKEN_VALUE \
+  --set serverFiles.prometheus.yml.scrape_configs[3].consul_sd_configs[0].authorization.credentials=$PROMETHEUS_CONSUL_TOKEN_VALUE
+```
+
+## Upgrade/Changes
+Обновление настроек и чарта делаем так:
+
+В начале сравниваем содержимое `values-override.yaml` с исходным файлом `values.yaml`
+(ссылка на исходную версию зафиксирована в заголовке `values-override.yaml`) через diff (IDE лучше).
+Или с новой версией `values.yaml` в Helm-чарте.
+Вносим необходимые правки, новые опции или добавляем секреты.
+
+Получаем из Vault секреты с паролями/токенами/сертификатами и выгружаем в окружение оболочки:
+```bash
+./prometheus_values_secrets_init.sh
+
+source .creds
+```
+
+Затем применяем:
+```bash
+helm upgrade -n prometheus prometheus-community/prometheus -f values-override.yaml \
+  --set serverFiles.prometheus.yml.scrape_configs[1].consul_sd_configs[0].authorization.credentials=$PROMETHEUS_CONSUL_TOKEN_VALUE \
+  --set serverFiles.prometheus.yml.scrape_configs[2].consul_sd_configs[0].authorization.credentials=$PROMETHEUS_CONSUL_TOKEN_VALUE \
+  --set serverFiles.prometheus.yml.scrape_configs[3].consul_sd_configs[0].authorization.credentials=$PROMETHEUS_CONSUL_TOKEN_VALUE
+```
--- a/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/argocd-apps-prometheus-app.yaml
+++ b/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/argocd-apps-prometheus-app.yaml
@@ -1,51 +0,0 @@
---
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: cloud-prometheus
-  namespace: argocd
-  finalizers:
-    - resources-finalizer.argocd.argoproj.io
-spec:
-  project: cloud-prometheus
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: cloud-prod
-  sources:
-    - repoURL: https://git.avroid.tech/K8s/k8s-configs.git
-      targetRevision: master
-      ref: values
-    - repoURL: https://nexus.avroid.tech/repository/devops-helm-proxy-helm/
-      chart: "prometheus-community/prometheus"
-      targetRevision: 27.5.1
-      helm:
-        valueFiles:
-          - $values/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/values-override.yaml
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - ApplyOutOfSyncOnly=true
-      - CreateNamespace=true
---
-apiVersion: argoproj.io/v1alpha1
-kind: AppProject
-metadata:
-  name: cloud-prometheus
-  namespace: argocd
-  # Finalizer that ensures that project is not deleted until it is not referenced by any application
-  finalizers:
-    - resources-finalizer.argocd.argoproj.io
-spec:
-  sourceRepos:
-    - https://git.avroid.tech/K8s/k8s-configs.git
-    - https://nexus.avroid.tech/repository/devops-helm-proxy-helm/
-  # Only permit applications to deploy to the guestbook namespace in the same cluster
-  destinations:
-    - namespace: cloud-prod
-      server: https://kubernetes.default.svc
-  # Deny all cluster-scoped resources from being created, except for Namespace
-  clusterResourceWhitelist:
-    - group: ''
-      kind: Namespace
--- a/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/argocd-apps-prometheus-secret.yaml
+++ b/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/argocd-apps-prometheus-secret.yaml
@@ -1,13 +0,0 @@
---
-apiVersion: v1
-kind: Secret
-metadata:
-  labels:
-    app.kubernetes.io/managed-by: argocd
-  name: prometheus-secret
-  namespace: cloud-prod
-  annotations:
-    avp.kubernetes.io/path: "team-devops/data/services/monitoring/k8s/cloud-prod/prometheus"
-type: Opaque
-stringData:
-  consul_secret: <consul-secret>
--- a/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/argocd-apps-prometheus-network-policy.yaml
+++ b/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/argocd-apps-prometheus-network-policy.yaml
--- a/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/prometheus_values_secrets_init.sh
+++ b/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/prometheus_values_secrets_init.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+set -e
+
+rm -rf .creds
+
+prometheus_consul_token_value=$(vault kv get team-devops//services/monitoring/k8s/cloud-prod/prometheus | grep consul.secret | awk '{print $2}')
+
+cat > .creds << EOF
+export PROMETHEUS_CONSUL_TOKEN_VALUE='${prometheus_consul_token_value}'
+EOF
+
+echo "Run:"
+echo '  1. source .creds'
+echo '  2. kubectl -n cloud-prod apply -f prometheus-network-policy.yaml'
--- a/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/values-override.yaml
+++ b/clusters/k8s-avroid-office.prod.local/namespaces/cloud-prod/monitoring/prometheus/values-override.yaml
@@ -138,7 +138,7 @@ serverFiles:
            tags: [ k8s_postgres_exporter ]
            services: [ monitoring_k8s_postgres_exporter ]
            authorization:
-              credentials_file: /etc/secrets/consul_secret
+              credentials: consul_secret
        relabel_configs:
          - source_labels: [ __meta_consul_service_metadata_metrics_path ]
            target_label: __metrics_path__
@@ -165,7 +165,7 @@ serverFiles:
            tags: [ k8s_redis_exporter ]
            services: [ monitoring_k8s_redis_exporter ]
            authorization:
-              credentials: /etc/secrets/consul_secret
+              credentials: consul_secret
        relabel_configs:
          - source_labels: [ __meta_consul_service_metadata_metrics_path ]
            target_label: __metrics_path__
@@ -187,7 +187,7 @@ serverFiles:
            tags: [ k8s_patroni_exporter ]
            services: [ monitoring_k8s_patroni_exporter ]
            authorization:
-              credentials: /etc/secrets/consul_secret
+              credentials: consul_secret
        relabel_configs:
          - source_labels: [ __meta_consul_service_metadata_metrics_path ]
            target_label: __metrics_path__