From bde6e3fc4bf099fc54cbb32854c12feea3e930ad Mon Sep 17 00:00:00 2001 From: Ilya Zaharenkov Date: Fri, 7 Mar 2025 12:51:23 +0300 Subject: [PATCH] [DO-1452] add build and deploy jobs from msg-auth (!98) Co-authored-by: Ilya Zaharenkov Co-authored-by: Denis Patrakeev Co-authored-by: Rustam Tagaev Reviewed-on: https://git.avroid.tech/DevOps/jenkins-pipelines/pulls/98 Reviewed-by: Denis Patrakeev Reviewed-by: Aleksandr Vodyanov Reviewed-by: Rustam Tagaev --- .../jobs/Cloud/Apps-Backend/msg_auth.groovy | 92 +++++++++++++ .../Deploy/Backend/msg_auth_deploy.groovy | 87 ++++++++++++ pipelines/Cloud/Apps-Backend/msg-auth.groovy | 104 ++++++++++++++ .../Deploy/Backend/msg-auth-deploy.groovy | 129 ++++++++++++++++++ 4 files changed, 412 insertions(+) create mode 100644 jobs-dsl/jobs/Cloud/Apps-Backend/msg_auth.groovy create mode 100644 jobs-dsl/jobs/Cloud/Deploy/Backend/msg_auth_deploy.groovy create mode 100644 pipelines/Cloud/Apps-Backend/msg-auth.groovy create mode 100644 pipelines/Cloud/Deploy/Backend/msg-auth-deploy.groovy diff --git a/jobs-dsl/jobs/Cloud/Apps-Backend/msg_auth.groovy b/jobs-dsl/jobs/Cloud/Apps-Backend/msg_auth.groovy new file mode 100644 index 0000000..80623b9 --- /dev/null +++ b/jobs-dsl/jobs/Cloud/Apps-Backend/msg_auth.groovy @@ -0,0 +1,92 @@ +multibranchPipelineJob('Cloud/Apps-Backend/msg-auth') { + description('msg-auth') + displayName('msg-auth') + factory { + remoteJenkinsFileWorkflowBranchProjectFactory { + localMarker('') + matchBranches(true) + fallbackBranch('master') + lookupInParameters(false) + remoteJenkinsFile('pipelines/Cloud/Apps-Backend/msg-auth.groovy') + remoteJenkinsFileSCM { + gitSCM { + userRemoteConfigs { + userRemoteConfig { + url("${JENKINS_GIT_REPOSITORY_URL}/DevOps/jenkins-pipelines.git") + credentialsId("${JENKINS_GIT_CREDENTIALS_HTTP}") + name('') + refspec('') + } + } + gitTool('') + browser {} + } + } + } + } + + branchSources { + branchSource { + source { + giteaSCMSource { + serverUrl("${JENKINS_GIT_REPOSITORY_URL}") + repoOwner('Apps-Backend') + repository('msg-auth') + credentialsId("${JENKINS_GIT_CREDENTIALS_HTTP}") + id('Apps-Backend/msg-auth') + traits { + giteaBranchDiscovery { + // 1 Exclude branches that are also filed as PRs + // 2 Only branches that are also filed as PRs + // 3 Only branches that are also filed as PRs or main + // 4 All branches + strategyId(1) + } + + giteaPullRequestDiscovery { + // 1 Merging the pull request with the current target branch revision + // 2 The current pull request revision + // 3 Both the current pull request revision and the pull request merged with + // the current target branch revision + strategyId(2) + } + + giteaForkDiscovery { + // 1 Merging the pull request with the current target branch revision + // 2 The current pull request revision + // 3 Both the current pull request revision and the pull request merged with + // the current target branch revision + strategyId(2) + trust { + giteaTrustContributors() + } + } + + giteaTagDiscovery() + pruneStaleBranch() + pruneStaleTag() + + refSpecs { + templates { + refSpecTemplate { + value('+refs/heads/*:refs/remotes/@{remote}/*') + } + } + } + } + } + } + } + } + orphanedItemStrategy { + discardOldItems { + numToKeep(20) + } + + defaultOrphanedItemStrategy { + pruneDeadBranches(true) + numToKeepStr('10') + daysToKeepStr('10') + } + } +} diff --git a/jobs-dsl/jobs/Cloud/Deploy/Backend/msg_auth_deploy.groovy b/jobs-dsl/jobs/Cloud/Deploy/Backend/msg_auth_deploy.groovy new file mode 100644 index 0000000..d77baa7 --- /dev/null +++ b/jobs-dsl/jobs/Cloud/Deploy/Backend/msg_auth_deploy.groovy @@ -0,0 +1,87 @@ +pipelineJob('Cloud/Deploy/Backend/msg-auth-deploy') { + logRotator { + numToKeep(10) + artifactNumToKeep(10) + } + + parameters { + choice { + name('ENV') + choices(['DEV']) + description('Select one of environments') + } + choiceParameter { + name('APP_VERSION') + randomName('') + filterable(true) + filterLength(1) + script { + groovyScript { + fallbackScript { + script('') + sandbox(false) + } + script { + sandbox(false) + script( +""" +import groovy.json.JsonSlurperClassic +import groovy.json.model.* +import com.cloudbees.plugins.credentials.CredentialsProvider +import com.cloudbees.plugins.credentials.common.StandardUsernamePasswordCredentials +import java.util.regex.Pattern + +def createGetHttpClient(String url, String jenkinsCreds) { + def jenkinsCredentials = CredentialsProvider.lookupCredentials(StandardUsernamePasswordCredentials.class) + def credentials = jenkinsCredentials.findResult { it.id == jenkinsCreds ? it : null } + String auth = 'robot' + '\$' + 'ci' + ":" + credentials.password + String encodedAuth = Base64.getEncoder().encodeToString(auth.getBytes("utf-8")) + String authHeaderValue = "Basic " + encodedAuth + def httpClient = new URL(url).openConnection() as HttpURLConnection + httpClient.setRequestMethod('GET') + httpClient.setRequestProperty("Authorization", authHeaderValue) + httpClient.setRequestProperty("Accept", "application/json") + return httpClient +} + +String harborApiUrl = "https://${JENKINS_DOCKER_REGISTRY}/api/v2.0/projects/cloud/repositories/msg-auth/" + + "artifacts?page=1&page_size=100&with_tag=true&sort=-push_time" + +def httpClientHarbor = createGetHttpClient(harborApiUrl, "${JENKINS_HARBOR_CREDENTIALS}") +httpClientHarbor.connect() + +List imageVersions = [] +def harborResponse = new JsonSlurperClassic().parseText(httpClientHarbor.inputStream.text) + +harborResponse.each { image -> + image.tags.each { tag -> + imageVersions.add(tag.name) + } +} + +return imageVersions +""" + ) + } + } + } + choiceType('PT_SINGLE_SELECT') + } + + } + + definition { + cpsScm { + scm { + git { + remote { + url("${JENKINS_GIT_REPOSITORY_URL}/DevOps/jenkins-pipelines.git") + credentials("${JENKINS_GIT_CREDENTIALS_HTTP}") + } + branch('master') + } + } + scriptPath('pipelines/Cloud/Deploy/Backend/msg-auth-deploy.groovy') + } + } +} diff --git a/pipelines/Cloud/Apps-Backend/msg-auth.groovy b/pipelines/Cloud/Apps-Backend/msg-auth.groovy new file mode 100644 index 0000000..a058ae0 --- /dev/null +++ b/pipelines/Cloud/Apps-Backend/msg-auth.groovy @@ -0,0 +1,104 @@ +@Library('shared-lib') _ + +import tech.avroid.kube.PodTemplates +import tech.avroid.scm.Git + +properties([ + buildDiscarder(logRotator(daysToKeepStr: '10', + numToKeepStr: '10')), + disableConcurrentBuilds() +]) + +Git git = new Git(this, env.JENKINS_GIT_CREDENTIALS_SSH) + +String repoPath = 'Apps-Backend/msg-auth.git' + +Map envBranch = [ + 'DEV': 'develop', + // 'TEST': 'test' +] + +String dockerGroup = 'cloud' +String dockerProject = 'msg-auth' +String projectSettingFile = 'pyproject.toml' +Map gitVars = [:] +String tag = '' +String branch = git.getBranch() +String publishBranch = envBranch.find { it.value == branch }?.value + +Map configuration = [ + vaultUrl: env.JENKINS_VAULT_URL, + vaultCredentialId: env.JENKINS_VAULT_TOKEN, + engineVersion: 2 +] + +List dockerCreds = [ + [path: 'team-devops/services/registry/Harbor/harbor.avroid.tech', engineVersion: 2, + secretValues: + [ + [vaultKey: 'service.user.jenkins.ci.login'], + [vaultKey: 'service.user.ci.token'], + ] + ] +] + +slaveTemplates = new PodTemplates(this, env.JENKINS_DOCKER_REGISTRY, ["${env.JENKINS_K8S_HARBOR_SECRET}"]) + +slaveTemplates.jnlp { + slaveTemplates.poetry { + slaveTemplates.docker { + node(POD_LABEL){ + + stage('Download sources') { + gitVars = git.clone([urlRepo: "${env.JENKINS_GIT_REPOSITORY_SSH_URL}/${repoPath}", + branch: branch]) + } + + stage('prepare app'){ + container('poetry'){ + sh 'make setup' + } + } + + withVault([configuration: configuration, vaultSecrets: dockerCreds]) { + String ciUser = getProperty('service.user.jenkins.ci.login') + String dockerToken = getProperty('service.user.ci.token') + + container('docker'){ + Map props = readTOML file: projectSettingFile + + String version = props.tool.poetry.version + tag = "${version}-${gitVars.GIT_COMMIT.take(5)}" + currentBuild.description = tag + + String imageName = "${env.JENKINS_DOCKER_REGISTRY}/" + + "${dockerGroup}/${dockerProject}:${tag}" + + docker.withRegistry("https://${env.JENKINS_DOCKER_REGISTRY}", env.JENKINS_HARBOR_CREDENTIALS) { + stage('build image and push'){ + Object buildImage = docker.build(imageName, + "--build-arg PIP_INDEX_URL=${env.JENKINS_PIP_INDEX_URL} -f Dockerfile ." + ) + + if (publishBranch != null && publishBranch.contains(branch)) { + buildImage.push() + } + } + } + + stage('deploy application'){ + if (publishBranch != null && publishBranch.contains(branch)){ + build job: 'Cloud/Deploy/Backend/msg-auth-deploy', + parameters: [ + string(name: 'ENV', value: envBranch.find { it.value == branch }?.key ), + string(name: 'APP_VERSION', value: tag) + ], + wait: false + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/pipelines/Cloud/Deploy/Backend/msg-auth-deploy.groovy b/pipelines/Cloud/Deploy/Backend/msg-auth-deploy.groovy new file mode 100644 index 0000000..96d0bf2 --- /dev/null +++ b/pipelines/Cloud/Deploy/Backend/msg-auth-deploy.groovy @@ -0,0 +1,129 @@ +@Library('shared-lib') _ + +import tech.avroid.kube.PodTemplates +import tech.avroid.scm.Git +import tech.avroid.jenkins.Notifications +import tech.avroid.jenkins.Jenkins + +String repositoryName = 'msg-auth' // Replace with your Harbor repository name +String k8sAppName = repositoryName +Map envBranch = [ + 'DEV': 'develop', + // 'TEST': 'test' +] + +properties([ + buildDiscarder(logRotator(artifactNumToKeepStr: '10', + numToKeepStr: '10')), + disableConcurrentBuilds(), + parameters([ + choice( + name: 'ENV', + choices: envBranch.keySet().toList(), + description: 'Select one of environments' + ), + [$class: 'ChoiceParameter', + choiceType: 'PT_SINGLE_SELECT', + filterLength: 1, + filterable: true, + name: 'APP_VERSION', + script: [$class: 'GroovyScript', + script: [sandbox: false, script: """ +import groovy.json.JsonSlurperClassic +import groovy.json.model.* +import com.cloudbees.plugins.credentials.CredentialsProvider +import com.cloudbees.plugins.credentials.common.StandardUsernamePasswordCredentials +import java.util.regex.Pattern + +def createGetHttpClient(String url, String jenkinsCreds) { + def jenkinsCredentials = CredentialsProvider.lookupCredentials(StandardUsernamePasswordCredentials.class) + def credentials = jenkinsCredentials.findResult { it.id == jenkinsCreds ? it : null } + String auth = 'robot' + '\$' + 'ci' + ":" + credentials.password + String encodedAuth = Base64.getEncoder().encodeToString(auth.getBytes("utf-8")) + String authHeaderValue = "Basic " + encodedAuth + def httpClient = new URL(url).openConnection() as HttpURLConnection + httpClient.setRequestMethod('GET') + httpClient.setRequestProperty("Authorization", authHeaderValue) + httpClient.setRequestProperty("Accept", "application/json") + return httpClient +} + +String harborApiUrl = "https://${JENKINS_DOCKER_REGISTRY}/api/v2.0/projects/cloud/repositories/msg-auth/" + + "artifacts?page=1&page_size=100&with_tag=true&sort=-push_time" + +def httpClientHarbor = createGetHttpClient(harborApiUrl, "${JENKINS_HARBOR_CREDENTIALS}") +httpClientHarbor.connect() + +List imageVersions = [] +def harborResponse = new JsonSlurperClassic().parseText(httpClientHarbor.inputStream.text) + +harborResponse.each { image -> + image.tags.each { tag -> + imageVersions.add(tag.name) + } +} + +return imageVersions +"""]], + ] + ]) + +]) +String repoPath = "Apps-Backend/${repositoryName}.git" +String branch = envBranch.find { it.key == params.ENV }?.value +String valuesPath = ".helm/values.${params.ENV.toLowerCase()}.yaml" +String namespace = "tavro-cloud-${params.ENV.toLowerCase()}" + + +String helmChart = repositoryName +String helmRepoPath = "avroid/${helmChart}" +String helmRepo = "${env.JENKINS_NEXUS_URL}/repository/devops-helm-release" +println(branch) +Git git = new Git(this, env.JENKINS_GIT_CREDENTIALS_SSH) +PodTemplates slaveTemplates = new PodTemplates(this, env.JENKINS_DOCKER_REGISTRY, + ["${env.JENKINS_K8S_HARBOR_SECRET}"], + 'avroid-office') + +slaveTemplates.jnlp { + slaveTemplates.helm { + try { + node(POD_LABEL){ + stage('get repo with values'){ + gitVars = git.clone([urlRepo: "${env.JENKINS_GIT_REPOSITORY_SSH_URL}/${repoPath}", + branch: branch]) + } + container('helm'){ + stage('deploy'){ + echo "Deploying version ${env.APP_VERSION} to ${namespace} namespace" + sh """#!/bin/sh + helm repo add avroid ${helmRepo} + helm -n ${namespace} upgrade -f ${valuesPath} \ + --set image.tag=${env.APP_VERSION} \ + --install ${k8sAppName} ${helmRepoPath} \ + --wait + """ + currentBuild.description = "ENV: ${params.ENV} TAG: ${env.APP_VERSION}" + } + } + } + } catch(err) { + errorMessage = err.getMessage() + + println 'ERROR: ' + errorMessage + + currentBuild.result = 'FAILURE' + + String currentBuildUser = Jenkins.GetCurrentBuildUser(script: this) + String emailSubject = "${currentBuild.currentResult}. " + + "Pipeline task: ${currentBuild.fullDisplayName}" + + Notifications.email( + script: this, + subject: emailSubject, + errorString: errorMessage, + recipientProviders: [], + to: "${currentBuildUser}@avroid.team" + ) + } + } +} \ No newline at end of file