[DO-272] readme

Reviewed-on: https://git.avroid.tech/DevOps/scripts/pulls/10

.gitignore

@@ -0,0 +1,9 @@
+.idea
+.vscode
+.vagrant
+.venv
+venv
+__pycache__
+*~
+**/.DS_Store
+._*

CODEOWNERS

@@ -0,0 +1,8 @@
+# All .yaml/.yml files are owned by CI/CD config managment team:
+# https://eva.avroid.tech/project/Document/DOC-000640#sotrudniki
+
+# Assign users to DOCS:
+README.md @stanislav.gabenov
+
+# Assign multiple users to all files in the project (fallback):
+* @DevOps/svc-gitea-devops-adm

README.md

@@ -0,0 +1,8 @@
+## Репозиторий для хранения скриптов автоматизации для различных сервисов
+
+Для хранения использовать следующую структуру:
+
+    <SERVICE>                       # Общее назначение сервиса где применяется скрипт
+    |___<local_path_on_host>        # (optional) Путь до скрипта на файловой системе
+    |   |___*.some_script_file      # 
+    |___README.md

Zulip/ldap-create-users-zulip.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Requires Python 3.6+
+
+import zulip
+import ldap
+
+
+# setup a function to correctly extract attribute values from the ldap results:
+def getAttribute(data, aName):
+	if aName in data[0][1]:
+		v = data[0][1][aName][0].decode('utf-8', 'ignore')
+		return v
+	return ""
+
+
+# Pass the path to your zuliprc file here. (must be an organization admin credential/apikey
+#  to create users, even bots with super knight-ed access can't create users)
+client = zulip.Client(config_file="~/zuliprc")
+
+# Get all users in the zulip realm
+allmembers = client.get_members()
+
+# extract just the 'members' list from the results:
+goodmembers = allmembers['members']
+
+# create the comparison list:
+zuliplist = []
+
+# for each loop through the goodmembers list of dictionary user entries and store in the zuliplist
+for d in goodmembers:
+	# add each discovered email address to the zulip list:
+	zuliplist.append(d['email'])
+
+# turn the list into a set:
+zulipset = set(zuliplist)
+
+# LDAP endpoint connection:
+LDAP_URL = "ldaps://ds01.avroid.tech"
+LDAP_USER = "uid=svc_ipa,cn=users,cn=accounts,dc=avroid,dc=tech"
+LDAP_PASSWORD = "xxxxxxxxxxxxx"
+LDAP_BASEDN = "cn=users,cn=accounts,dc=avroid,dc=tech"
+LDAP_SEARCH_FILTER = "(&(objectClass=inetorgperson)(memberOf=cn=org-avroid-all,cn=groups,cn=accounts,dc=avroid,dc=tech))"
+LDAP_RETRIEVE_ATTRIBUTES = ["krbPrincipalName", "uid", "DisplayName"]
+
+
+# Define the LDAP lookup using parameters from above
+l = ldap.initialize(LDAP_URL)
+l.simple_bind_s(LDAP_USER, LDAP_PASSWORD)
+searchScope = ldap.SCOPE_SUBTREE
+
+# initialize i to zero to use it as a counter
+i = 0
+
+# harvest the ldap results and check the list against the zulipset, create the user in zulip if missing:
+try:
+	ldap_result_id = l.search(LDAP_BASEDN, searchScope, LDAP_SEARCH_FILTER, LDAP_RETRIEVE_ATTRIBUTES)
+	result_set = []
+	while 1:
+		i = i + 1
+		result_type, result_data = l.result(ldap_result_id, 0)
+		if not result_data:
+			break
+		else:
+			upn = getAttribute(result_data,"krbPrincipalName")     # Email | krbPrincipalName
+			shortname = getAttribute(result_data,"uid")  # UID
+			displayname = getAttribute(result_data,"DisplayName")          # DisplayName
+
+			if upn in zulipset:
+				print("user found in zulip already : "+upn)
+			else:
+				print("user needs to be added to zulip : "+upn)
+
+				# Create the user with a 'fake' password via the zulip library
+				# (the password field is required, but only SAMLauth backend is enabled, so this password "can't" be used)
+				request = {
+					'email': upn,
+					'password': 'fakeComplexpasswordThatWillNeverbeused!4858025279014',
+					'full_name': displayname,
+					'short_name': shortname
+				}
+				result = client.create_user(request)
+				print(result)
+
+# print any ldap errors that may have occurred during the query:
+except ldap.LDAPError as e:
+
+	print(e)

backup_scripts/consul/pre_backup.sh

@@ -8,6 +8,6 @@ HOSTNAME=$(hostname)
 
 LEADER=$(consul operator raft list-peers | grep leader | cut -f 1 -d " ")
 
-[ "$LEADER" = "consul-vault-01" ] \
+[ "$LEADER" = "${HOSTNAME}" ] \
 && (echo "I'm leader" && echo "Create backup" && consul snapshot save "${BACKUP_DIR_PATH}/${BACKUP_FILE}" ) \
 || echo "I'm not leader"

backup_scripts/postgres/post_backup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+DIRBACKUP="/data/backups"
+
+if [ -d "${DIRBACKUP}" ]; then
+    rm -rf "${DIRBACKUP:?}/"*
+fi

backup_scripts/postgres/pre_backup.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+DATE=$(date +%F)
+DIRBACKUP="/data/backups"
+DBUSER="postgres"
+
+get_leader() {
+    curl -s 127.0.0.1:8008 | jq -j ".role"
+}
+
+if [ "$(get_leader)" != "master" ]; then
+    echo "I'm not leader. Exit"
+    exit 0
+fi
+
+getDbName() {
+    psql -lA -x -U ${DBUSER} -p 15432 -d postgres | grep Name | grep -v -P "template0|template1" | cut -d "|" -f 2
+}
+
+DBLIST=$(getDbName)
+
+createBackup() {
+    for db in $DBLIST; do
+        echo "===================================="
+        echo "START create backup database: ${db}"
+
+        pg_dump -Fc -U ${DBUSER} -p 15432 -d "${db}" >"${DIRBACKUP}/${db}-${DATE}".sql
+        
+        bzip2 -9 "${DIRBACKUP}/${db}-${DATE}".sql
+        
+        echo "END database bzip2: ${db}"
+    done
+
+    pg_dumpall -p 15432 -U postgres --roles-only > "${DIRBACKUP}/roles-${DATE}".sql
+}
+
+
+[[ $(ls -A ${DIRBACKUP}) ]] 2>/dev/null && echo "dir ${DIRBACKUP} not empty" && exit 123
+
+createBackup

backup_scripts/postgres/pre_backup_docker.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+DATE=$(date +%F)
+DIRBACKUP="/data/backups"
+DBUSER="postgres"
+
+getDbName() {
+    docker exec postgresql bash -c "psql -lA -x -U postgres -p 5432 -d postgres | \
+                                    grep Name | grep -v -P 'template0|template1' | cut -f 2  -d '|'"
+}
+
+DBLIST=$(getDbName)
+
+createBackup() {
+    for db in $DBLIST; do
+        echo "===================================="
+        echo "START create backup database: ${db}"
+
+        docker exec postgresql bash -c "pg_dump -Fc -U ${DBUSER} -p 5432 -d ${db}" > "${DIRBACKUP}/${db}-${DATE}".sql
+        
+        bzip2 -9 "${DIRBACKUP}/${db}-${DATE}".sql
+
+        echo "END database bzip2: ${db}"
+    done
+}
+
+[[ $(ls -A ${DIRBACKUP}) ]] 2>/dev/null && echo "dir ${DIRBACKUP} not empty" && exit 123
+
+createBackup

backup_scripts/postgres/pre_backup_for_backup_test.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+DATE=$(date +%F)
+DIRBACKUP="/data/backups"
+DBUSER="postgres"
+TEST_BACKUP_SERVER="h-license-manager-backup-server.avroid.cloud"
+
+get_leader() {
+    curl -s 127.0.0.1:8008 | jq -j ".role"
+}
+
+if [ "$(get_leader)" != "master" ]; then
+    echo "I'm not leader. Exit"
+    exit 0
+fi
+
+getDbName() {
+    psql -lA -x -U ${DBUSER} -p 15432 -d postgres | grep Name | grep -v -P "template0|template1" | cut -d "|" -f 2
+}
+
+DBLIST=$(getDbName)
+
+createBackup() {
+    for db in $DBLIST; do
+        echo "===================================="
+        echo "START create backup database: ${db}"
+
+        pg_dump -Fc -U ${DBUSER} -p 15432 -d "${db}" >"${DIRBACKUP}/${db}-${DATE}".sql
+        
+        bzip2 -9 "${DIRBACKUP}/${db}-${DATE}".sql
+        
+        copyBackupToTestServer "${DIRBACKUP}/${db}-${DATE}".sql.bz2 ${TEST_BACKUP_SERVER}
+
+        echo "END database bzip2: ${db}"
+    done
+
+    pg_dumpall -p 15432 -U postgres --roles-only > "${DIRBACKUP}/roles-${DATE}".sql
+    copyBackupToTestServer  "${DIRBACKUP}/roles-${DATE}".sql ${TEST_BACKUP_SERVER}
+}
+
+copyBackupToTestServer() {
+	rsync $1 $2::backup || true
+}
+
+[[ $(ls -A ${DIRBACKUP}) ]] 2>/dev/null && echo "dir ${DIRBACKUP} not empty" && exit 123
+
+createBackup
+copyBackupToTestServer

imap_exporter/Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.12.3-alpine
+
+COPY ["requirements.txt", "imap_exporter.py", "./"]
+
+RUN apk add openssl
+
+RUN python3 -m pip install -r requirements.txt --no-cache-dir
+
+EXPOSE 9119
+
+CMD ["python", "-u", "imap_exporter.py"]

imap_exporter/imap_exporter.py

@@ -0,0 +1,38 @@
+import time
+import os
+import re
+import subprocess
+from prometheus_client import start_http_server, Gauge
+
+
+rs = r"CN=\*.avroid.tech"
+host = os.environ.get("IMAP_HOST", "imap-app.avroid.tech:993")
+re_search = os.environ.get("RE_SEARCH", rs)
+
+# Create a metric to track time spent and requests made.
+g = Gauge('connection_success', 'Подключение к imap серверу', ["imap_host"])
+
+def get_connection_status(imap_host):
+    try:
+        response = subprocess.check_output(f"openssl s_client -connect {imap_host} > /dev/null < /dev/null", shell=True,stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as e:
+        response = e.output
+
+    text_response = response.decode('utf-8')
+
+    if re.search(re_search, text_response):
+        print(f"Connection to {host} successfull")
+        connection_status = 1
+    else:
+        print(f"Connection to {host} failure")
+        connection_status = 0
+    return connection_status
+
+
+if __name__ == '__main__':
+    # Start up the server to expose the metrics.
+    start_http_server(9120)
+    while True:
+        cs = get_connection_status(host)
+        g.labels(host).set(cs)
+        time.sleep(5)

imap_exporter/requirements.txt

@@ -0,0 +1 @@
+prometheus_client==0.20.0

imap_exporter/restart_imap.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Переменная для tmp lock-файла
+LOCK_FILE="/tmp/restart_imap.lock"
+
+# Функция для проверки блокировки
+check_lock() {
+    if [ -f "$LOCK_FILE" ]; then
+        echo "$(date) Скрипт уже запущен, выход..." >> /var/log/restart_imap.log
+        exit 1
+    fi
+}
+
+# Функция для установки блокировки
+set_lock() {
+    touch "$LOCK_FILE"
+}
+
+# Функция для удаления блокировки
+remove_lock() {
+    rm -f "$LOCK_FILE"
+}
+
+# Основной код скрипта
+check_lock
+set_lock
+
+openssl s_client -connect imap-app.avroid.tech:993 > /dev/null < /dev/null > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+    echo "$(date) Restart imap docker container" >> /var/log/restart_imap.log
+    docker restart imap
+fi
+
+remove_lock