From 6b1b32a22eaea6319fe78f4144a9d2622841f1f1 Mon Sep 17 00:00:00 2001 From: Timothe Date: Wed, 22 Jun 2022 09:25:37 +0200 Subject: [PATCH 01/35] first commit (adding glab pipe gen and creating artifact function) --- .../azure-devops/pipeline_generator.sh | 13 ++ .../pipelines/common/pipeline_generator.lib | 22 +-- .../pipelines/github/pipeline_generator.sh | 17 +- .../pipelines/gitlab/pipeline_generator.sh | 180 ++++++++++++++++++ 4 files changed, 211 insertions(+), 21 deletions(-) create mode 100644 scripts/pipelines/gitlab/pipeline_generator.sh diff --git a/scripts/pipelines/azure-devops/pipeline_generator.sh b/scripts/pipelines/azure-devops/pipeline_generator.sh index b680b6a2c..b9cfd03ec 100644 --- a/scripts/pipelines/azure-devops/pipeline_generator.sh +++ b/scripts/pipelines/azure-devops/pipeline_generator.sh @@ -54,6 +54,17 @@ function obtainHangarPath { hangarPath=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && cd ../../.. && pwd ) } +function addAdditionalArtifact { + # Check if an extra artifact to store is supplied. + if test ! -z "$artifactPath" + then + # Add the extra step to the YAML. + cat "${hangarPath}/${commonTemplatesPath}/store-extra-path.yml" >> "${localDirectory}/${pipelinePath}/${yamlFile}" + else + echo "The '-a' flag has not been set, skipping the step to add additional artifact." + fi +} + function createPipeline { echo -e "${green}Generating the pipeline from the YAML template..." echo -ne ${white} @@ -150,6 +161,8 @@ createNewBranch copyYAMLFile +addAdditionalArtifact + copyCommonScript type copyScript &> /dev/null && copyScript diff --git a/scripts/pipelines/common/pipeline_generator.lib b/scripts/pipelines/common/pipeline_generator.lib index 55fab26a0..9061d8152 100644 --- a/scripts/pipelines/common/pipeline_generator.lib +++ b/scripts/pipelines/common/pipeline_generator.lib @@ -107,6 +107,9 @@ function checkInstallations { elif ([ "$provider" == "azure-devops" ] && ! [ -x "$(command -v az)" ]); then echo -e "${red}Error: Azure CLI is not installed." >&2 exit 127 + elif ([ "$provider" == "gitlab" ] && ! [ -x "$(command -v glab)" ]); then + echo -e "${red}Error: Azure CLI is not installed." >&2 + exit 127 fi # Check if Python is installed @@ -136,25 +139,6 @@ function copyYAMLFile { # We cannot use a variable in the definition of resource in the pipeline so we have to use a placeholder to replace it with the value we need commonEnvSubstList='${buildPipelineName} ${testPipelineName} ${qualityPipelineName} ${pipelineName}' envsubst "${commonEnvSubstList} ${specificEnvSubstList}" < "${hangarPath}/${templatesPath}/${yamlFile}.template" > "${localDirectory}/${pipelinePath}/${yamlFile}" - - # Check if an extra artifact to store is supplied. - if test ! -z "$artifactPath" - then - # Add the extra step to the YAML. - if [ "$provider" == "azure-devops" ] - then - cat "${hangarPath}/${commonTemplatesPath}/store-extra-path.yml" >> "${localDirectory}/${pipelinePath}/${yamlFile}" - elif [ "$provider" == "github" ] - then - storeExtraPathContent="\n - name: Publish Additional Output Artifact\n uses: actions\/upload-artifact@v3\n with:\n name: additional-pipeline-output\n path: \"\${{ env.artifactPath }}\"" - sed -i "s/# mark to insert step for additonal artifact #/$storeExtraPathContent\n/" "${localDirectory}/${pipelinePath}/${yamlFile}" - fi - else - if [ "$provider" == "github" ] - then - sed -i '/# mark to insert step for additonal artifact #/d' "${localDirectory}/${pipelinePath}/${yamlFile}" - fi - fi } function copyCommonScript { diff --git a/scripts/pipelines/github/pipeline_generator.sh b/scripts/pipelines/github/pipeline_generator.sh index 274f0b753..f64c735e7 100644 --- a/scripts/pipelines/github/pipeline_generator.sh +++ b/scripts/pipelines/github/pipeline_generator.sh @@ -52,6 +52,19 @@ function obtainHangarPath { hangarPath=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && cd ../../.. && pwd ) } +function addAdditionalArtifact { + # Check if an extra artifact to store is supplied. + if test ! -z "$artifactPath" + then + # Add the extra step to the YAML. + storeExtraPathContent="\n - name: Publish Additional Output Artifact\n uses: actions\/upload-artifact@v3\n with:\n name: additional-pipeline-output\n path: \"\${{ env.artifactPath }}\"" + sed -i "s/# mark to insert step for additonal artifact #/$storeExtraPathContent\n/" "${localDirectory}/${pipelinePath}/${yamlFile}" + else + echo "The '-a' flag has not been set, skipping the step to add additional artifact." + sed -i '/# mark to insert step for additonal artifact #/d' "${localDirectory}/${pipelinePath}/${yamlFile}" + fi +} + # Function that adds the variables to be used in the pipeline. function addCommonPipelineVariables { if test -z "${artifactPath}" @@ -137,6 +150,8 @@ type addPipelineVariables &> /dev/null && addPipelineVariables copyYAMLFile +addAdditionalArtifact + copyCommonScript type copyScript &> /dev/null && copyScript @@ -148,6 +163,4 @@ commitCommonFiles type commitFiles &> /dev/null && commitFiles -# createPipeline - createPR diff --git a/scripts/pipelines/gitlab/pipeline_generator.sh b/scripts/pipelines/gitlab/pipeline_generator.sh new file mode 100644 index 000000000..0e90f3d58 --- /dev/null +++ b/scripts/pipelines/gitlab/pipeline_generator.sh @@ -0,0 +1,180 @@ +#!/bin/bash +set -e +FLAGS=$(getopt -a --options c:n:d:a:b:l:i:u:p:hw --long "config-file:,pipeline-name:,local-directory:,artifact-path:,target-branch:,language:,build-pipeline-name:,sonar-url:,sonar-token:,image-name:,registry-user:,registry-password:,resource-group:,storage-account:,storage-container:,cluster-name:,s3-bucket:,s3-key-path:,quality-pipeline-name:,dockerfile:,test-pipeline-name:,aws-access-key:,aws-secret-access-key:,aws-region:,help" -- "$@") + +eval set -- "$FLAGS" +while true; do + case "$1" in + -c | --config-file) configFile=$2; shift 2;; + -n | --pipeline-name) export pipelineName=$2; shift 2;; + -d | --local-directory) localDirectory=$2; shift 2;; + -a | --artifact-path) artifactPath=$2; shift 2;; + -b | --target-branch) targetBranch=$2; shift 2;; + -l | --language) language=$2; shift 2;; + --build-pipeline-name) export buildPipelineName=$2; shift 2;; + --sonar-url) sonarUrl=$2; shift 2;; + --sonar-token) sonarToken=$2; shift 2;; + -i | --image-name) imageName=$2; shift 2;; + -u | --registry-user) dockerUser=$2; shift 2;; + -p | --registry-password) dockerPassword=$2; shift 2;; + --resource-group) resourceGroupName=$2; shift 2;; + --storage-account) storageAccountName=$2; shift 2;; + --storage-container) storageContainerName=$2; shift 2;; + --cluster-name) clusterName=$2; shift 2;; + --s3-bucket) s3Bucket=$2; shift 2;; + --s3-key-path) s3KeyPath=$2; shift 2;; + --quality-pipeline-name) export qualityPipelineName=$2; shift 2;; + --test-pipeline-name) export testPipelineName=$2; shift 2;; + --dockerfile) dockerFile=$2; shift 2;; + --aws-access-key) awsAccessKey="$2"; shift 2;; + --aws-secret-access-key) awsSecretAccessKey="$2"; shift 2;; + --aws-region) awsRegion="$2"; shift 2;; + -h | --help) help="true"; shift 1;; + -w) webBrowser="true"; shift 1;; + --) shift; break;; + esac +done + +# Colours for the messages. +white='\e[1;37m' +green='\e[1;32m' +red='\e[0;31m' + +# Common var +commonTemplatesPath="scripts/pipelines/gitlab/templates/common" # Path for common files of the pipelines +pipelinePath=".pipelines" # Path to the pipelines. +scriptFilePath=".pipelines/scripts" # Path to the scripts. +gitlabCiFile=".gitlab-ci.yml" +export provider="gitlab" + +function obtainHangarPath { + + # This line goes to the script directory independent of wherever the user is and then jumps 3 directories back to get the path + hangarPath=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && cd ../../.. && pwd ) +} + +function addAdditionalArtifact { + # Check if an extra artifact to store is supplied. + if test ! -z "$artifactPath" + then + # Add the extra step to the YAML. + grep " artifacts:" "${localDirectory}/${pipelinePath}/${yamlFile}" && storeExtraPathContent=" - \"$artifactPath\"" + grep " artifacts:" "${localDirectory}/${pipelinePath}/${yamlFile}" || storeExtraPathContent="\n artifacts:\n - name: addtionalArtifactOutput\n paths:\n - \"$artifactPath\"" + sed -i "s/# mark to insert step for additonal artifact #/$storeExtraPathContent\n/" "${localDirectory}/${pipelinePath}/${yamlFile}" + else + echo "The '-a' flag has not been set, skipping the step to add additional artifact." + sed -i '/# mark to insert step for additonal artifact #/d' "${localDirectory}/${pipelinePath}/${yamlFile}" + fi +} + +# Function that adds the variables to be used in the pipeline. +function addCommonPipelineVariables { + if test -z "${artifactPath}" + then + echo "Skipping creation of the variable artifactPath as the flag has not been used." + # Delete the commentary to set the artifactPath input/var + sed -i '/# mark to insert additional artifact env var #/d' "${localDirectory}/${pipelinePath}/${yamlFile}" + else + # add the input for the additional artifact + grep "variables:" "${localDirectory}/${pipelinePath}/${yamlFile}" > /dev/null && textArtifactPathVar=" artifactPath: ${artifactPath//\//\\/}" + grep "variables:" "${localDirectory}/${pipelinePath}/${yamlFile}" > /dev/null || textArtifactPathVar="variables:\n artifactPath: \"${artifactPath//\//\\/}\"" + sed -i "s/# mark to insert additional artifact env var #/$textArtifactPathVar/" "${localDirectory}/${pipelinePath}/${yamlFile}" + fi +} + +function addCiFile { + echo -e "${green}Copying and commiting the gitlab ci file." + echo -ne ${white} + + cp "${hangarPath}/${commonTemplatesPath}/${gitlabCiFile}" "${localDirectory}/${gitlabCiFile}" + testCommit=$(git status) + if echo "$testCommit" | grep "nothing to commit, working tree clean" > /dev/null + then + echo "gilab-ci file already present with same content, nothing to commit." + else + git add "${gitlabCiFile}" -f + git commit -m "adding gitlab-ci.yml" + git push + fi +} + +function createPR { + # Check if a target branch is supplied. + if test -z "$targetBranch" + then + # No branch specified in the parameters, no Pull Request is created, the code will be stored in the current branch. + echo -e "${green}No branch specified to do the Pull Request, changes left in the ${sourceBranch} branch." + exit + else + echo -e "${green}Creating a Pull Request..." + echo -ne "${white}" + repoURL=$(git config --get remote.origin.url) + repoNameWithGit="${repoURL/https:\/\/github.com\/}" + repoName="${repoNameWithGit/.git}" + # Create the Pull Request to merge into the specified branch. + #debug + echo "glab mr create -b \"$targetBranch\" -d \"merge request $sourceBranch\" -s \"$sourceBranch\" -H \"${repoName}\" -t \"merge $sourceBranch\"" + pr=$(glab mr create -b "$targetBranch" -d "merge request $sourceBranch" -s "$sourceBranch" -H "${repoName}" -t "merge $sourceBranch") + + # trying to merge + if glab mr merge -s $(basename "$pr") -y + then + # Pull Request merged successfully. + echo -e "${green}Pull Request merged into $targetBranch branch successfully." + exit + else + # Check if the -w flag is activated. + if [[ "$webBrowser" == "true" ]] + then + # -w flag is activated and a page with the corresponding Pull Request is opened in the web browser. + echo -e "${green}Pull Request successfully created." + echo -e "${green}Opening the Pull Request on the web browser..." + python -m webbrowser "$pr" + exit + else + # -w flag is not activated and the URL to the Pull Request is shown in the console. + echo -e "${green}Pull Request successfully created." + echo -e "${green}To review the Pull Request and accept it, click on the following link:" + echo "${pr}" + exit + fi + fi + fi +} + + +obtainHangarPath + +# Load common functions +. "$hangarPath/scripts/pipelines/common/pipeline_generator.lib" + +if [[ "$help" == "true" ]]; then help; fi + +ensurePathFormat + +importConfigFile + +checkInstallations + +createNewBranch + +type addPipelineVariables &> /dev/null && addPipelineVariables + +copyYAMLFile + +addAdditionalArtifact + +copyCommonScript + +type copyScript &> /dev/null && copyScript + +# This function does not exists for the github pipeline generator at this moment, but I let the line with 'type' to keep the same structure as the others pipeline generator +type addCommonPipelineVariables &> /dev/null && addCommonPipelineVariables + +commitCommonFiles + +type commitFiles &> /dev/null && commitFiles + +addCiFile + +createPR From 54a04e1b6db732dca23b04099c5f920c3dd485b2 Mon Sep 17 00:00:00 2001 From: Timothe Date: Wed, 22 Jun 2022 09:27:29 +0200 Subject: [PATCH 02/35] adding gitlab-ci file --- .../pipelines/gitlab/templates/common/.gitlab-ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml diff --git a/scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml b/scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml new file mode 100644 index 000000000..34904840a --- /dev/null +++ b/scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml @@ -0,0 +1,12 @@ +include: + - '.pipelines/*.yml' + +stages: + - build + - test + - quality + - package + +default: + image: maven:3-jdk-11 + tags: ['docker_ruby'] \ No newline at end of file From edf91dcd7613ec179feedaaa5a732a185e922b9f Mon Sep 17 00:00:00 2001 From: Timothe Date: Thu, 7 Jul 2022 10:00:52 +0200 Subject: [PATCH 03/35] fixing some little errors --- scripts/pipelines/gitlab/pipeline_generator.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/pipelines/gitlab/pipeline_generator.sh b/scripts/pipelines/gitlab/pipeline_generator.sh index 0e90f3d58..3f04b4d29 100644 --- a/scripts/pipelines/gitlab/pipeline_generator.sh +++ b/scripts/pipelines/gitlab/pipeline_generator.sh @@ -58,8 +58,8 @@ function addAdditionalArtifact { if test ! -z "$artifactPath" then # Add the extra step to the YAML. - grep " artifacts:" "${localDirectory}/${pipelinePath}/${yamlFile}" && storeExtraPathContent=" - \"$artifactPath\"" - grep " artifacts:" "${localDirectory}/${pipelinePath}/${yamlFile}" || storeExtraPathContent="\n artifacts:\n - name: addtionalArtifactOutput\n paths:\n - \"$artifactPath\"" + grep " artifacts:" "${localDirectory}/${pipelinePath}/${yamlFile}" > /dev/null && storeExtraPathContent=" - \"$artifactPath\"" + grep " artifacts:" "${localDirectory}/${pipelinePath}/${yamlFile}" > /dev/null || storeExtraPathContent="\n artifacts:\n paths:\n - \"$artifactPath\"" sed -i "s/# mark to insert step for additonal artifact #/$storeExtraPathContent\n/" "${localDirectory}/${pipelinePath}/${yamlFile}" else echo "The '-a' flag has not been set, skipping the step to add additional artifact." @@ -109,7 +109,7 @@ function createPR { echo -e "${green}Creating a Pull Request..." echo -ne "${white}" repoURL=$(git config --get remote.origin.url) - repoNameWithGit="${repoURL/https:\/\/github.com\/}" + repoNameWithGit="${repoURL/https:\/\/gitlab.com\/}" repoName="${repoNameWithGit/.git}" # Create the Pull Request to merge into the specified branch. #debug From 82ff220ede3ec685a6d7f658e68a512ddc3a554e Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 5 Sep 2022 10:52:05 +0530 Subject: [PATCH 04/35] Add files via upload --- .../templates/common/aks/aks-pipeline.cfg | 60 ++++++ .../common/aks/aks-provisioning.yml.template | 173 ++++++++++++++++++ .../gitlab/templates/common/aks/obtain-dns.sh | 21 +++ .../kubernetes/install-nginx-ingress.sh | 4 + .../common/kubernetes/install-rancher.sh | 7 + 5 files changed, 265 insertions(+) create mode 100644 scripts/pipelines/gitlab/templates/common/aks/aks-pipeline.cfg create mode 100644 scripts/pipelines/gitlab/templates/common/aks/aks-provisioning.yml.template create mode 100644 scripts/pipelines/gitlab/templates/common/aks/obtain-dns.sh create mode 100644 scripts/pipelines/gitlab/templates/common/common/kubernetes/install-nginx-ingress.sh create mode 100644 scripts/pipelines/gitlab/templates/common/common/kubernetes/install-rancher.sh diff --git a/scripts/pipelines/gitlab/templates/common/aks/aks-pipeline.cfg b/scripts/pipelines/gitlab/templates/common/aks/aks-pipeline.cfg new file mode 100644 index 000000000..ef9c0c861 --- /dev/null +++ b/scripts/pipelines/gitlab/templates/common/aks/aks-pipeline.cfg @@ -0,0 +1,60 @@ +# Mandatory flags. +mandatoryFalgs="$pipelineName,$localDirectory,$clusterName,$storageContainerName," #$resourceGroupName,$storageAccountName,$storageContainerName," +# Path to the templates. +templatesPath="scripts/pipelines/github/templates/aks" +#Path to common kubernetes templates. +commonKubernetesPath="scripts/pipelines/github/templates/common/kubernetes" +# aks-provision YAML file name. +yamlFile=".gitlab-ci.yml" +# Source branch. +sourceBranch="feature/aks-provisioning" +# Path to terraform templates. +terraformTemplatesPath="scripts/environment-provisioning/azure/aks" +# Path to terraform scripts. +terraformPath=".terraform/aks" +# Default cluster operation. +operation="create" +# Install Rancher on AKS cluster. +if test -z $installRancher +then + installRancher=false +fi + +# Function that copies the necessary scripts into the directory. +function copyScript { + # Create .terraform/aks folder if it does not exist. + mkdir -p "${localDirectory}/${terraformPath}" + + # Copy the terraform files. + cd "${hangarPath}/${terraformTemplatesPath}" + cp * "${localDirectory}/${terraformPath}" + + # Copy the script for the DNS name into the directory. + cp "${hangarPath}/${templatesPath}/obtain-dns.sh" "${localDirectory}/${scriptFilePath}/obtain-dns.sh" + + # Copy the common files for kubernetes + cp "${hangarPath}/${commonKubernetesPath}"/*.sh "${localDirectory}/${scriptFilePath}" +} + +function commitFiles { + # Add the terraform files. + git add .terraform -f + + # Changing all files to be executable. + find .terraform -type f -name '*.sh' -exec git update-index --chmod=+x {} \; + + # Git commit and push it into the repository. + git commit -m "Adding the terraform files" + git push -u origin ${sourceBranch} +} + +# Function that adds the variables to be used in the pipeline. +function addPipelineVariables { + export clusterName + # export resourceGroupName + # export storageAccountName + # export storageContainerName + export installRancher + export operation + specificEnvSubstList='${clusterName} ${installRancher} ${operation}' #${resourceGroupName} ${storageAccountName} ${storageContainerName} +} diff --git a/scripts/pipelines/gitlab/templates/common/aks/aks-provisioning.yml.template b/scripts/pipelines/gitlab/templates/common/aks/aks-provisioning.yml.template new file mode 100644 index 000000000..4e18313d4 --- /dev/null +++ b/scripts/pipelines/gitlab/templates/common/aks/aks-provisioning.yml.template @@ -0,0 +1,173 @@ +default: + image: + name: ubuntu:latest + entrypoint: + - /usr/bin/env + - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +workflow: + rules: + - if: '$CI_PIPELINE_SOURCE == "web"' + when: always + - when: never + +variables: + CLUSTER_NAME: + value: $clusterName + description: "Name for the AWS EKS cluster to be created" + OPERATION: + value: $operation + description: "Operation to perform on cluster. Create or Destroy." + INSTALL_RANCHER: + value: $installRancher + description: "Installs Rancher on AKS when set to true." + #CLUSTER_NAME: devon-hangar + #OPERATION: create + TF_STATE_NAME: $storageContainerName #"TF_AKS_STATE_FILE" + TF_CACHE_KEY: default + TF_ROOT: "${CI_PROJECT_DIR}/azure/aks" + TF_USERNAME: ${GITLAB_USER_NAME} + TF_PASSWORD: ${GITLAB_PAT} + TF_ADDRESS: "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/terraform/state/${TF_STATE_NAME}" + TF_HTTP_ADDRESS: ${TF_ADDRESS} + TF_HTTP_LOCK_ADDRESS: ${TF_ADDRESS}/lock + TF_HTTP_LOCK_METHOD: POST + TF_HTTP_UNLOCK_ADDRESS: ${TF_ADDRESS}/lock + TF_HTTP_UNLOCK_METHOD: DELETE + TF_HTTP_USERNAME: ${TF_USERNAME} + TF_HTTP_PASSWORD: ${TF_PASSWORD} + TF_HTTP_RETRY_WAIT_MIN: 5 + +.Prerequisites_install: &Prerequisites_install + before_script: + - apt-get update + - apt-get install sudo -y + - apt-get install curl -y + - apt-get install zip -y + - apt-get install -y wget + +.configure_azcli: &login_azcli + # INSTALL AZCLI + - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + - az --version + - az login #--username "${AZURE_USERNAME}" --password "${AZURE_PASSWORD}" + +.packages: &configure_packages + # INSTALL KUBECTL + - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + - chmod +x ./kubectl + - mv ./kubectl /usr/local/bin/kubectl + # INSTALL HELM + - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + - chmod +x get_helm.sh + - DESIRED_VERSION=v3.9.0 ./get_helm.sh + +.install-terraform: &install-terraform + # INSTALL + - wget -nv https://releases.hashicorp.com/terraform/1.2.6/terraform_1.2.6_linux_amd64.zip + - unzip -qq terraform_1.2.6_linux_amd64.zip + - sudo mv terraform /usr/local/bin + +.configure_kubeconfig: &configure_kubeconfig + # - "curl 'https://gitlab.com/$CI_PROJECT_PATH/-/jobs/$CI_JOB_ID/artifacts/${CI_COMMIT_REF_NAME}/raw/kubeconfig?job=$CI_JOB_STAGE&JOB-TOKEN=$CI_JOB_TOKEN'" + - export KUBECONFIG=${KUBECONFIG} + +Provision: + <<: *Prerequisites_install + script: + - echo "hello $CLUSTER_NAME:$OPERATION, $TF_STATE_NAME" + - *install-terraform + - mkdir -p ${CI_PROJECT_DIR}/scripts + - cd ${CI_PROJECT_DIR}/scripts + - *login_azcli + - cd .. + - terraform --version + - cd ${TF_ROOT} + - terraform init -var cluster_name=${CLUSTER_NAME} + - terraform apply -var cluster_name=${CLUSTER_NAME} --auto-approve + - mv ${TF_ROOT}/kubeconfig ${CI_PROJECT_DIR} + - curl -s https://raw.githubusercontent.com/profclems/glab/trunk/scripts/install.sh | sudo sh + - glab auth login --token $GITLAB_PAT + - cat ${CI_PROJECT_DIR}/kubeconfig | glab variable set -g devon-hangar KUBECONFIG -t "file" + artifacts: + paths: + - "./kubeconfig" + cache: + key: "scripts" + paths: + - "${CI_PROJECT_DIR}/scripts" + rules: + - if: '$OPERATION == "create"' + when: always + +Install_nginx: + <<: *Prerequisites_install + needs: [Provision] + cache: + key: "scripts" + paths: + - "${CI_PROJECT_DIR}/scripts" + script: + - cd ${CI_PROJECT_DIR}/scripts + - *configure_packages + - *configure_kubeconfig + - chmod 755 install-nginx-ingress.sh + - ./install-nginx-ingress.sh + rules: + - if: '$OPERATION == "create"' + when: always + +Obtain_dns: + <<: *Prerequisites_install + script: + - cd ${CI_PROJECT_DIR}/scripts + - *configure_packages + - *login_azcli + - *configure_kubeconfig + # Obtain-dns + - chmod 755 obtain-dns.sh + - ./obtain-dns.sh ${CLUSTER_NAME} + - dnsname="${CLUSTER_NAME}.westeurope.cloudapp.azure.com" + # Create aks_dns_name variable + - curl -s https://raw.githubusercontent.com/profclems/glab/trunk/scripts/install.sh | sudo sh + - glab auth login --token $GITLAB_PAT + - glab variable set -g devon-hangar aks_dns_name -v "$dnsname" -t "env_var" + rules: + - if: '$OPERATION == "create"' + when: always + needs: + - Provision + #- Install_nginx + +Install-rancher: + <<: *Prerequisites_install + needs: + - Provision + - Obtain_dns + cache: + key: "scripts" + paths: + - "${CI_PROJECT_DIR}/scripts" + script: + - cd ${CI_PROJECT_DIR}/scripts + - *configure_packages + - *configure_kubeconfig + # INSTALL RANCHER + - chmod 755 install-rancher.sh + - ./install-rancher.sh ${aks_dns_name} + rules: + - if: '$INSTALL_RANCHER == "true"' + when: always + +Destroy-terraform: + <<: *Prerequisites_install + script: + - *install-terraform + - *configure_packages + - cd ${TF_ROOT} + - ls -lrta + - terraform init + - terraform apply -destroy -var cluster_name=${CLUSTER_NAME} --auto-approve + rules: + - if: '$OPERATION == "destroy"' + when: always diff --git a/scripts/pipelines/gitlab/templates/common/aks/obtain-dns.sh b/scripts/pipelines/gitlab/templates/common/aks/obtain-dns.sh new file mode 100644 index 000000000..567cc74a5 --- /dev/null +++ b/scripts/pipelines/gitlab/templates/common/aks/obtain-dns.sh @@ -0,0 +1,21 @@ +#!/bin/bash +ip="$(kubectl get svc nginx-ingress-nginx-ingress-controller --namespace nginx-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" + +while test -z "$ip" +do + sleep 5s + ip="$(kubectl get svc nginx-ingress-nginx-ingress-controller --namespace nginx-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" +done + +# Obtain the AKS cluster name +dnsname=$1 + +ipname=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$ip')].[name]" --output tsv) + +iprg=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$ip')].[resourceGroup]" --output tsv) + +az network public-ip update --resource-group "$iprg" --name "$ipname" --dns-name "$dnsname" + +dns="$(az network public-ip show --resource-group "$iprg" --name "$ipname" --query "[dnsSettings.fqdn]" --output tsv)" + +echo "$dns" diff --git a/scripts/pipelines/gitlab/templates/common/common/kubernetes/install-nginx-ingress.sh b/scripts/pipelines/gitlab/templates/common/common/kubernetes/install-nginx-ingress.sh new file mode 100644 index 000000000..f735dd93b --- /dev/null +++ b/scripts/pipelines/gitlab/templates/common/common/kubernetes/install-nginx-ingress.sh @@ -0,0 +1,4 @@ +#!/bin/bash +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo update +helm install nginx-ingress bitnami/nginx-ingress-controller --set ingressClassResource.default=true --set containerSecurityContext.allowPrivilegeEscalation=false --namespace nginx-ingress --create-namespace \ No newline at end of file diff --git a/scripts/pipelines/gitlab/templates/common/common/kubernetes/install-rancher.sh b/scripts/pipelines/gitlab/templates/common/common/kubernetes/install-rancher.sh new file mode 100644 index 000000000..eebe87ea9 --- /dev/null +++ b/scripts/pipelines/gitlab/templates/common/common/kubernetes/install-rancher.sh @@ -0,0 +1,7 @@ +#!/bin/bash +helm repo add rancher-latest https://releases.rancher.com/server-charts/latest +helm repo add jetstack https://charts.jetstack.io +helm repo add +kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.1/cert-manager.crds.yaml +helm install cert-manager "jetstack/cert-manager" --namespace cert-manager --create-namespace --version v1.5.1 +helm install rancher "rancher-latest/rancher" --namespace cattle-system --create-namespace --set hostname="$1" \ No newline at end of file From 295c33754df77ec44bf1eca309b029ece2ee5bc9 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 5 Sep 2022 10:53:25 +0530 Subject: [PATCH 05/35] Update main.tf --- scripts/environment-provisioning/azure/aks/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/environment-provisioning/azure/aks/main.tf b/scripts/environment-provisioning/azure/aks/main.tf index 618c0ea74..e015b1bc9 100644 --- a/scripts/environment-provisioning/azure/aks/main.tf +++ b/scripts/environment-provisioning/azure/aks/main.tf @@ -6,7 +6,7 @@ terraform { } } - backend "azurerm" {} + backend "http" {} } provider "azurerm" { @@ -34,4 +34,4 @@ resource "azurerm_kubernetes_cluster" "cluster" { identity { type = "SystemAssigned" } -} \ No newline at end of file +} From df6cf1d209bd3ebfdaaa5307cb3c1c937f01b65a Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 5 Sep 2022 10:56:24 +0530 Subject: [PATCH 06/35] Delete scripts/pipelines/gitlab/templates/common/aks directory --- .../templates/common/aks/aks-pipeline.cfg | 60 ------ .../common/aks/aks-provisioning.yml.template | 173 ------------------ .../gitlab/templates/common/aks/obtain-dns.sh | 21 --- 3 files changed, 254 deletions(-) delete mode 100644 scripts/pipelines/gitlab/templates/common/aks/aks-pipeline.cfg delete mode 100644 scripts/pipelines/gitlab/templates/common/aks/aks-provisioning.yml.template delete mode 100644 scripts/pipelines/gitlab/templates/common/aks/obtain-dns.sh diff --git a/scripts/pipelines/gitlab/templates/common/aks/aks-pipeline.cfg b/scripts/pipelines/gitlab/templates/common/aks/aks-pipeline.cfg deleted file mode 100644 index ef9c0c861..000000000 --- a/scripts/pipelines/gitlab/templates/common/aks/aks-pipeline.cfg +++ /dev/null @@ -1,60 +0,0 @@ -# Mandatory flags. -mandatoryFalgs="$pipelineName,$localDirectory,$clusterName,$storageContainerName," #$resourceGroupName,$storageAccountName,$storageContainerName," -# Path to the templates. -templatesPath="scripts/pipelines/github/templates/aks" -#Path to common kubernetes templates. -commonKubernetesPath="scripts/pipelines/github/templates/common/kubernetes" -# aks-provision YAML file name. -yamlFile=".gitlab-ci.yml" -# Source branch. -sourceBranch="feature/aks-provisioning" -# Path to terraform templates. -terraformTemplatesPath="scripts/environment-provisioning/azure/aks" -# Path to terraform scripts. -terraformPath=".terraform/aks" -# Default cluster operation. -operation="create" -# Install Rancher on AKS cluster. -if test -z $installRancher -then - installRancher=false -fi - -# Function that copies the necessary scripts into the directory. -function copyScript { - # Create .terraform/aks folder if it does not exist. - mkdir -p "${localDirectory}/${terraformPath}" - - # Copy the terraform files. - cd "${hangarPath}/${terraformTemplatesPath}" - cp * "${localDirectory}/${terraformPath}" - - # Copy the script for the DNS name into the directory. - cp "${hangarPath}/${templatesPath}/obtain-dns.sh" "${localDirectory}/${scriptFilePath}/obtain-dns.sh" - - # Copy the common files for kubernetes - cp "${hangarPath}/${commonKubernetesPath}"/*.sh "${localDirectory}/${scriptFilePath}" -} - -function commitFiles { - # Add the terraform files. - git add .terraform -f - - # Changing all files to be executable. - find .terraform -type f -name '*.sh' -exec git update-index --chmod=+x {} \; - - # Git commit and push it into the repository. - git commit -m "Adding the terraform files" - git push -u origin ${sourceBranch} -} - -# Function that adds the variables to be used in the pipeline. -function addPipelineVariables { - export clusterName - # export resourceGroupName - # export storageAccountName - # export storageContainerName - export installRancher - export operation - specificEnvSubstList='${clusterName} ${installRancher} ${operation}' #${resourceGroupName} ${storageAccountName} ${storageContainerName} -} diff --git a/scripts/pipelines/gitlab/templates/common/aks/aks-provisioning.yml.template b/scripts/pipelines/gitlab/templates/common/aks/aks-provisioning.yml.template deleted file mode 100644 index 4e18313d4..000000000 --- a/scripts/pipelines/gitlab/templates/common/aks/aks-provisioning.yml.template +++ /dev/null @@ -1,173 +0,0 @@ -default: - image: - name: ubuntu:latest - entrypoint: - - /usr/bin/env - - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - -workflow: - rules: - - if: '$CI_PIPELINE_SOURCE == "web"' - when: always - - when: never - -variables: - CLUSTER_NAME: - value: $clusterName - description: "Name for the AWS EKS cluster to be created" - OPERATION: - value: $operation - description: "Operation to perform on cluster. Create or Destroy." - INSTALL_RANCHER: - value: $installRancher - description: "Installs Rancher on AKS when set to true." - #CLUSTER_NAME: devon-hangar - #OPERATION: create - TF_STATE_NAME: $storageContainerName #"TF_AKS_STATE_FILE" - TF_CACHE_KEY: default - TF_ROOT: "${CI_PROJECT_DIR}/azure/aks" - TF_USERNAME: ${GITLAB_USER_NAME} - TF_PASSWORD: ${GITLAB_PAT} - TF_ADDRESS: "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/terraform/state/${TF_STATE_NAME}" - TF_HTTP_ADDRESS: ${TF_ADDRESS} - TF_HTTP_LOCK_ADDRESS: ${TF_ADDRESS}/lock - TF_HTTP_LOCK_METHOD: POST - TF_HTTP_UNLOCK_ADDRESS: ${TF_ADDRESS}/lock - TF_HTTP_UNLOCK_METHOD: DELETE - TF_HTTP_USERNAME: ${TF_USERNAME} - TF_HTTP_PASSWORD: ${TF_PASSWORD} - TF_HTTP_RETRY_WAIT_MIN: 5 - -.Prerequisites_install: &Prerequisites_install - before_script: - - apt-get update - - apt-get install sudo -y - - apt-get install curl -y - - apt-get install zip -y - - apt-get install -y wget - -.configure_azcli: &login_azcli - # INSTALL AZCLI - - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - - az --version - - az login #--username "${AZURE_USERNAME}" --password "${AZURE_PASSWORD}" - -.packages: &configure_packages - # INSTALL KUBECTL - - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - - chmod +x ./kubectl - - mv ./kubectl /usr/local/bin/kubectl - # INSTALL HELM - - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - - chmod +x get_helm.sh - - DESIRED_VERSION=v3.9.0 ./get_helm.sh - -.install-terraform: &install-terraform - # INSTALL - - wget -nv https://releases.hashicorp.com/terraform/1.2.6/terraform_1.2.6_linux_amd64.zip - - unzip -qq terraform_1.2.6_linux_amd64.zip - - sudo mv terraform /usr/local/bin - -.configure_kubeconfig: &configure_kubeconfig - # - "curl 'https://gitlab.com/$CI_PROJECT_PATH/-/jobs/$CI_JOB_ID/artifacts/${CI_COMMIT_REF_NAME}/raw/kubeconfig?job=$CI_JOB_STAGE&JOB-TOKEN=$CI_JOB_TOKEN'" - - export KUBECONFIG=${KUBECONFIG} - -Provision: - <<: *Prerequisites_install - script: - - echo "hello $CLUSTER_NAME:$OPERATION, $TF_STATE_NAME" - - *install-terraform - - mkdir -p ${CI_PROJECT_DIR}/scripts - - cd ${CI_PROJECT_DIR}/scripts - - *login_azcli - - cd .. - - terraform --version - - cd ${TF_ROOT} - - terraform init -var cluster_name=${CLUSTER_NAME} - - terraform apply -var cluster_name=${CLUSTER_NAME} --auto-approve - - mv ${TF_ROOT}/kubeconfig ${CI_PROJECT_DIR} - - curl -s https://raw.githubusercontent.com/profclems/glab/trunk/scripts/install.sh | sudo sh - - glab auth login --token $GITLAB_PAT - - cat ${CI_PROJECT_DIR}/kubeconfig | glab variable set -g devon-hangar KUBECONFIG -t "file" - artifacts: - paths: - - "./kubeconfig" - cache: - key: "scripts" - paths: - - "${CI_PROJECT_DIR}/scripts" - rules: - - if: '$OPERATION == "create"' - when: always - -Install_nginx: - <<: *Prerequisites_install - needs: [Provision] - cache: - key: "scripts" - paths: - - "${CI_PROJECT_DIR}/scripts" - script: - - cd ${CI_PROJECT_DIR}/scripts - - *configure_packages - - *configure_kubeconfig - - chmod 755 install-nginx-ingress.sh - - ./install-nginx-ingress.sh - rules: - - if: '$OPERATION == "create"' - when: always - -Obtain_dns: - <<: *Prerequisites_install - script: - - cd ${CI_PROJECT_DIR}/scripts - - *configure_packages - - *login_azcli - - *configure_kubeconfig - # Obtain-dns - - chmod 755 obtain-dns.sh - - ./obtain-dns.sh ${CLUSTER_NAME} - - dnsname="${CLUSTER_NAME}.westeurope.cloudapp.azure.com" - # Create aks_dns_name variable - - curl -s https://raw.githubusercontent.com/profclems/glab/trunk/scripts/install.sh | sudo sh - - glab auth login --token $GITLAB_PAT - - glab variable set -g devon-hangar aks_dns_name -v "$dnsname" -t "env_var" - rules: - - if: '$OPERATION == "create"' - when: always - needs: - - Provision - #- Install_nginx - -Install-rancher: - <<: *Prerequisites_install - needs: - - Provision - - Obtain_dns - cache: - key: "scripts" - paths: - - "${CI_PROJECT_DIR}/scripts" - script: - - cd ${CI_PROJECT_DIR}/scripts - - *configure_packages - - *configure_kubeconfig - # INSTALL RANCHER - - chmod 755 install-rancher.sh - - ./install-rancher.sh ${aks_dns_name} - rules: - - if: '$INSTALL_RANCHER == "true"' - when: always - -Destroy-terraform: - <<: *Prerequisites_install - script: - - *install-terraform - - *configure_packages - - cd ${TF_ROOT} - - ls -lrta - - terraform init - - terraform apply -destroy -var cluster_name=${CLUSTER_NAME} --auto-approve - rules: - - if: '$OPERATION == "destroy"' - when: always diff --git a/scripts/pipelines/gitlab/templates/common/aks/obtain-dns.sh b/scripts/pipelines/gitlab/templates/common/aks/obtain-dns.sh deleted file mode 100644 index 567cc74a5..000000000 --- a/scripts/pipelines/gitlab/templates/common/aks/obtain-dns.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -ip="$(kubectl get svc nginx-ingress-nginx-ingress-controller --namespace nginx-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" - -while test -z "$ip" -do - sleep 5s - ip="$(kubectl get svc nginx-ingress-nginx-ingress-controller --namespace nginx-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" -done - -# Obtain the AKS cluster name -dnsname=$1 - -ipname=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$ip')].[name]" --output tsv) - -iprg=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$ip')].[resourceGroup]" --output tsv) - -az network public-ip update --resource-group "$iprg" --name "$ipname" --dns-name "$dnsname" - -dns="$(az network public-ip show --resource-group "$iprg" --name "$ipname" --query "[dnsSettings.fqdn]" --output tsv)" - -echo "$dns" From bd1195adff48d59b0d292c6cbbcf303375c20cbe Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 5 Sep 2022 10:56:39 +0530 Subject: [PATCH 07/35] Delete scripts/pipelines/gitlab/templates/common/common/kubernetes directory --- .../common/common/kubernetes/install-nginx-ingress.sh | 4 ---- .../templates/common/common/kubernetes/install-rancher.sh | 7 ------- 2 files changed, 11 deletions(-) delete mode 100644 scripts/pipelines/gitlab/templates/common/common/kubernetes/install-nginx-ingress.sh delete mode 100644 scripts/pipelines/gitlab/templates/common/common/kubernetes/install-rancher.sh diff --git a/scripts/pipelines/gitlab/templates/common/common/kubernetes/install-nginx-ingress.sh b/scripts/pipelines/gitlab/templates/common/common/kubernetes/install-nginx-ingress.sh deleted file mode 100644 index f735dd93b..000000000 --- a/scripts/pipelines/gitlab/templates/common/common/kubernetes/install-nginx-ingress.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -helm repo add bitnami https://charts.bitnami.com/bitnami -helm repo update -helm install nginx-ingress bitnami/nginx-ingress-controller --set ingressClassResource.default=true --set containerSecurityContext.allowPrivilegeEscalation=false --namespace nginx-ingress --create-namespace \ No newline at end of file diff --git a/scripts/pipelines/gitlab/templates/common/common/kubernetes/install-rancher.sh b/scripts/pipelines/gitlab/templates/common/common/kubernetes/install-rancher.sh deleted file mode 100644 index eebe87ea9..000000000 --- a/scripts/pipelines/gitlab/templates/common/common/kubernetes/install-rancher.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -helm repo add rancher-latest https://releases.rancher.com/server-charts/latest -helm repo add jetstack https://charts.jetstack.io -helm repo add -kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.1/cert-manager.crds.yaml -helm install cert-manager "jetstack/cert-manager" --namespace cert-manager --create-namespace --version v1.5.1 -helm install rancher "rancher-latest/rancher" --namespace cattle-system --create-namespace --set hostname="$1" \ No newline at end of file From 6fb30ff777e68b3dc185cb4d2905bfd1630c5c50 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 5 Sep 2022 10:57:33 +0530 Subject: [PATCH 08/35] Add files via upload --- .../gitlab/templates/aks/aks-pipeline.cfg | 60 ++++++ .../aks/aks-provisioning.yml.template | 173 ++++++++++++++++++ .../gitlab/templates/aks/obtain-dns.sh | 21 +++ .../kubernetes/install-nginx-ingress.sh | 4 + .../common/kubernetes/install-rancher.sh | 7 + 5 files changed, 265 insertions(+) create mode 100644 scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg create mode 100644 scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template create mode 100644 scripts/pipelines/gitlab/templates/aks/obtain-dns.sh create mode 100644 scripts/pipelines/gitlab/templates/common/kubernetes/install-nginx-ingress.sh create mode 100644 scripts/pipelines/gitlab/templates/common/kubernetes/install-rancher.sh diff --git a/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg b/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg new file mode 100644 index 000000000..ef9c0c861 --- /dev/null +++ b/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg @@ -0,0 +1,60 @@ +# Mandatory flags. +mandatoryFalgs="$pipelineName,$localDirectory,$clusterName,$storageContainerName," #$resourceGroupName,$storageAccountName,$storageContainerName," +# Path to the templates. +templatesPath="scripts/pipelines/github/templates/aks" +#Path to common kubernetes templates. +commonKubernetesPath="scripts/pipelines/github/templates/common/kubernetes" +# aks-provision YAML file name. +yamlFile=".gitlab-ci.yml" +# Source branch. +sourceBranch="feature/aks-provisioning" +# Path to terraform templates. +terraformTemplatesPath="scripts/environment-provisioning/azure/aks" +# Path to terraform scripts. +terraformPath=".terraform/aks" +# Default cluster operation. +operation="create" +# Install Rancher on AKS cluster. +if test -z $installRancher +then + installRancher=false +fi + +# Function that copies the necessary scripts into the directory. +function copyScript { + # Create .terraform/aks folder if it does not exist. + mkdir -p "${localDirectory}/${terraformPath}" + + # Copy the terraform files. + cd "${hangarPath}/${terraformTemplatesPath}" + cp * "${localDirectory}/${terraformPath}" + + # Copy the script for the DNS name into the directory. + cp "${hangarPath}/${templatesPath}/obtain-dns.sh" "${localDirectory}/${scriptFilePath}/obtain-dns.sh" + + # Copy the common files for kubernetes + cp "${hangarPath}/${commonKubernetesPath}"/*.sh "${localDirectory}/${scriptFilePath}" +} + +function commitFiles { + # Add the terraform files. + git add .terraform -f + + # Changing all files to be executable. + find .terraform -type f -name '*.sh' -exec git update-index --chmod=+x {} \; + + # Git commit and push it into the repository. + git commit -m "Adding the terraform files" + git push -u origin ${sourceBranch} +} + +# Function that adds the variables to be used in the pipeline. +function addPipelineVariables { + export clusterName + # export resourceGroupName + # export storageAccountName + # export storageContainerName + export installRancher + export operation + specificEnvSubstList='${clusterName} ${installRancher} ${operation}' #${resourceGroupName} ${storageAccountName} ${storageContainerName} +} diff --git a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template new file mode 100644 index 000000000..4e18313d4 --- /dev/null +++ b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template @@ -0,0 +1,173 @@ +default: + image: + name: ubuntu:latest + entrypoint: + - /usr/bin/env + - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +workflow: + rules: + - if: '$CI_PIPELINE_SOURCE == "web"' + when: always + - when: never + +variables: + CLUSTER_NAME: + value: $clusterName + description: "Name for the AWS EKS cluster to be created" + OPERATION: + value: $operation + description: "Operation to perform on cluster. Create or Destroy." + INSTALL_RANCHER: + value: $installRancher + description: "Installs Rancher on AKS when set to true." + #CLUSTER_NAME: devon-hangar + #OPERATION: create + TF_STATE_NAME: $storageContainerName #"TF_AKS_STATE_FILE" + TF_CACHE_KEY: default + TF_ROOT: "${CI_PROJECT_DIR}/azure/aks" + TF_USERNAME: ${GITLAB_USER_NAME} + TF_PASSWORD: ${GITLAB_PAT} + TF_ADDRESS: "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/terraform/state/${TF_STATE_NAME}" + TF_HTTP_ADDRESS: ${TF_ADDRESS} + TF_HTTP_LOCK_ADDRESS: ${TF_ADDRESS}/lock + TF_HTTP_LOCK_METHOD: POST + TF_HTTP_UNLOCK_ADDRESS: ${TF_ADDRESS}/lock + TF_HTTP_UNLOCK_METHOD: DELETE + TF_HTTP_USERNAME: ${TF_USERNAME} + TF_HTTP_PASSWORD: ${TF_PASSWORD} + TF_HTTP_RETRY_WAIT_MIN: 5 + +.Prerequisites_install: &Prerequisites_install + before_script: + - apt-get update + - apt-get install sudo -y + - apt-get install curl -y + - apt-get install zip -y + - apt-get install -y wget + +.configure_azcli: &login_azcli + # INSTALL AZCLI + - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + - az --version + - az login #--username "${AZURE_USERNAME}" --password "${AZURE_PASSWORD}" + +.packages: &configure_packages + # INSTALL KUBECTL + - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + - chmod +x ./kubectl + - mv ./kubectl /usr/local/bin/kubectl + # INSTALL HELM + - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + - chmod +x get_helm.sh + - DESIRED_VERSION=v3.9.0 ./get_helm.sh + +.install-terraform: &install-terraform + # INSTALL + - wget -nv https://releases.hashicorp.com/terraform/1.2.6/terraform_1.2.6_linux_amd64.zip + - unzip -qq terraform_1.2.6_linux_amd64.zip + - sudo mv terraform /usr/local/bin + +.configure_kubeconfig: &configure_kubeconfig + # - "curl 'https://gitlab.com/$CI_PROJECT_PATH/-/jobs/$CI_JOB_ID/artifacts/${CI_COMMIT_REF_NAME}/raw/kubeconfig?job=$CI_JOB_STAGE&JOB-TOKEN=$CI_JOB_TOKEN'" + - export KUBECONFIG=${KUBECONFIG} + +Provision: + <<: *Prerequisites_install + script: + - echo "hello $CLUSTER_NAME:$OPERATION, $TF_STATE_NAME" + - *install-terraform + - mkdir -p ${CI_PROJECT_DIR}/scripts + - cd ${CI_PROJECT_DIR}/scripts + - *login_azcli + - cd .. + - terraform --version + - cd ${TF_ROOT} + - terraform init -var cluster_name=${CLUSTER_NAME} + - terraform apply -var cluster_name=${CLUSTER_NAME} --auto-approve + - mv ${TF_ROOT}/kubeconfig ${CI_PROJECT_DIR} + - curl -s https://raw.githubusercontent.com/profclems/glab/trunk/scripts/install.sh | sudo sh + - glab auth login --token $GITLAB_PAT + - cat ${CI_PROJECT_DIR}/kubeconfig | glab variable set -g devon-hangar KUBECONFIG -t "file" + artifacts: + paths: + - "./kubeconfig" + cache: + key: "scripts" + paths: + - "${CI_PROJECT_DIR}/scripts" + rules: + - if: '$OPERATION == "create"' + when: always + +Install_nginx: + <<: *Prerequisites_install + needs: [Provision] + cache: + key: "scripts" + paths: + - "${CI_PROJECT_DIR}/scripts" + script: + - cd ${CI_PROJECT_DIR}/scripts + - *configure_packages + - *configure_kubeconfig + - chmod 755 install-nginx-ingress.sh + - ./install-nginx-ingress.sh + rules: + - if: '$OPERATION == "create"' + when: always + +Obtain_dns: + <<: *Prerequisites_install + script: + - cd ${CI_PROJECT_DIR}/scripts + - *configure_packages + - *login_azcli + - *configure_kubeconfig + # Obtain-dns + - chmod 755 obtain-dns.sh + - ./obtain-dns.sh ${CLUSTER_NAME} + - dnsname="${CLUSTER_NAME}.westeurope.cloudapp.azure.com" + # Create aks_dns_name variable + - curl -s https://raw.githubusercontent.com/profclems/glab/trunk/scripts/install.sh | sudo sh + - glab auth login --token $GITLAB_PAT + - glab variable set -g devon-hangar aks_dns_name -v "$dnsname" -t "env_var" + rules: + - if: '$OPERATION == "create"' + when: always + needs: + - Provision + #- Install_nginx + +Install-rancher: + <<: *Prerequisites_install + needs: + - Provision + - Obtain_dns + cache: + key: "scripts" + paths: + - "${CI_PROJECT_DIR}/scripts" + script: + - cd ${CI_PROJECT_DIR}/scripts + - *configure_packages + - *configure_kubeconfig + # INSTALL RANCHER + - chmod 755 install-rancher.sh + - ./install-rancher.sh ${aks_dns_name} + rules: + - if: '$INSTALL_RANCHER == "true"' + when: always + +Destroy-terraform: + <<: *Prerequisites_install + script: + - *install-terraform + - *configure_packages + - cd ${TF_ROOT} + - ls -lrta + - terraform init + - terraform apply -destroy -var cluster_name=${CLUSTER_NAME} --auto-approve + rules: + - if: '$OPERATION == "destroy"' + when: always diff --git a/scripts/pipelines/gitlab/templates/aks/obtain-dns.sh b/scripts/pipelines/gitlab/templates/aks/obtain-dns.sh new file mode 100644 index 000000000..567cc74a5 --- /dev/null +++ b/scripts/pipelines/gitlab/templates/aks/obtain-dns.sh @@ -0,0 +1,21 @@ +#!/bin/bash +ip="$(kubectl get svc nginx-ingress-nginx-ingress-controller --namespace nginx-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" + +while test -z "$ip" +do + sleep 5s + ip="$(kubectl get svc nginx-ingress-nginx-ingress-controller --namespace nginx-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" +done + +# Obtain the AKS cluster name +dnsname=$1 + +ipname=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$ip')].[name]" --output tsv) + +iprg=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$ip')].[resourceGroup]" --output tsv) + +az network public-ip update --resource-group "$iprg" --name "$ipname" --dns-name "$dnsname" + +dns="$(az network public-ip show --resource-group "$iprg" --name "$ipname" --query "[dnsSettings.fqdn]" --output tsv)" + +echo "$dns" diff --git a/scripts/pipelines/gitlab/templates/common/kubernetes/install-nginx-ingress.sh b/scripts/pipelines/gitlab/templates/common/kubernetes/install-nginx-ingress.sh new file mode 100644 index 000000000..f735dd93b --- /dev/null +++ b/scripts/pipelines/gitlab/templates/common/kubernetes/install-nginx-ingress.sh @@ -0,0 +1,4 @@ +#!/bin/bash +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo update +helm install nginx-ingress bitnami/nginx-ingress-controller --set ingressClassResource.default=true --set containerSecurityContext.allowPrivilegeEscalation=false --namespace nginx-ingress --create-namespace \ No newline at end of file diff --git a/scripts/pipelines/gitlab/templates/common/kubernetes/install-rancher.sh b/scripts/pipelines/gitlab/templates/common/kubernetes/install-rancher.sh new file mode 100644 index 000000000..eebe87ea9 --- /dev/null +++ b/scripts/pipelines/gitlab/templates/common/kubernetes/install-rancher.sh @@ -0,0 +1,7 @@ +#!/bin/bash +helm repo add rancher-latest https://releases.rancher.com/server-charts/latest +helm repo add jetstack https://charts.jetstack.io +helm repo add +kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.1/cert-manager.crds.yaml +helm install cert-manager "jetstack/cert-manager" --namespace cert-manager --create-namespace --version v1.5.1 +helm install rancher "rancher-latest/rancher" --namespace cattle-system --create-namespace --set hostname="$1" \ No newline at end of file From aebbf7c516a9eb03f6e634d5b90bb12d78d9291b Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Thu, 8 Sep 2022 10:39:04 +0530 Subject: [PATCH 09/35] update --- .../setup-aks-provisioning-pipeline.asciidoc | 89 ++++++++++++++ .../setup-aks-provisioning-pipeline.asciidoc | 7 ++ .../setup-aks-provisioning-pipeline.asciidoc | 112 ++++++++++++++++++ .../setup-aks-provisioning-pipeline.asciidoc | 7 ++ .../pipelines/gitlab/pipeline_generator.sh | 3 +- .../gitlab/templates/aks/aks-pipeline.cfg | 10 +- .../aks/aks-provisioning.yml.template | 10 +- .../gitlab/templates/common/.gitlab-ci.yml | 16 +-- 8 files changed, 235 insertions(+), 19 deletions(-) create mode 100644 documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc create mode 100644 documentation/src/azure-devops/setup-aks-provisioning-pipeline.asciidoc create mode 100644 documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc create mode 100644 documentation/src/gitlab/setup-aks-provisioning-pipeline.asciidoc diff --git a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc new file mode 100644 index 000000000..07d0dd68d --- /dev/null +++ b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc @@ -0,0 +1,89 @@ += Setting up a Azure AKS provisioning pipeline on Azure DevOps + +In this section we will create a pipeline which will provision an Azure AKS cluster. This pipeline will be configured to be manually triggered by the user. As part of AKS cluster provisioning, a NGINX Ingress controller is deployed and a variable group with the name `aks-variables` is created, which contains, among others, the DNS name of the Ingress controller, that you you will need to add as CNAME record on the domains used in your application Ingress manifest files. Refer to the appendix to retrieve the DNS name of the Ingress controller independently. + +The creation of the pipeline will follow the project workflow, so a new branch named `feature/aks-provisioning` will be created, the YAML file for the pipeline and the terraform files for creating the cluster will be pushed to it. + +Then, a Pull Request (PR) will be created in order to merge the new branch into the appropiate branch (provided in `-b` flag). The PR will be automatically merged if the repository policies are met. If the merge is not possible, either the PR URL will be shown as output, or it will be opened in your web browser if using `-w` flag. + +The script located at `/scripts/pipelines/azure-devops/pipeline_generator.sh` will automatically create this new branch, create the AKS provisioning pipeline based on the YAML template, create the Pull Request and, if it is possible, merge this new branch into the specified branch. + +== Prerequisites + +* Install the https://marketplace.visualstudio.com/items?itemName=ms-devlabs.custom-terraform-tasks[Terraform extension] for Azure DevOps. +* Create a https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#create-a-service-connection[service connection] to Azure Resource Manager and name it `aks-connection`. If you already have a service connection available or you need a specific connection name, please update `aks-pipeline.cfg` accordingly. +* An Azure resource group in the desired cluster location (e.g. `westeurope`). You can use an existing one or create a new one with the following command: + +``` +az group create -n -l +``` + +* An Azure storage account within the previous resource group. You can use an existing one or create a new one with the following command: + +``` +az storage account create -n -g -l +``` + +* An Azure storage container in Azure within the previous storage account. You can use an existing one or create a new one with the following command: + +``` +az storage container create -n --account-name +``` + +* This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled the latest changes with `git pull`). + +== Creating the pipeline using provided script + +Before executing the script you will need to customize some input variables about the environment. To do so, you can either edit `terraform.tfvars` file or take advantage of the `set-terraform-variables.sh` script located at `/scripts/environment-provisioning/azure/aks`, which allows you to create or update values for the required variables, passing them as flags. As a full example: + +``` +./set-terraform-variables.sh --cluster_name --location --resource_group_name --instance_type --worker_node_count --dns_prefix +``` + +=== Usage +``` +pipeline_generator.sh \ + -c \ + -n \ + -d \ + --resource-group \ + --storage-account \ + --storage-container \ + [-b ] \ + [-w] +``` + +NOTE: The config file for the AKS provisioning pipeline is located at `/scripts/pipelines/azure-devops/templates/aks/aks-pipeline.cfg`. + +=== Flags +``` +-c, --config-file [Required] Configuration file containing pipeline definition. +-n, --pipeline-name [Required] Name that will be set to the pipeline. +-d, --local-directory [Required] Local directory of your project (the path should always be using '/' and not '\'). + --resource-group [Required] Name of the resource group for the cluster. + --storage--account [Required] Name of the storage account for the cluster. + --storage-container [Required] Name of the storage container where the Terraform state of the cluster will be stored. +-b, --target-branch Name of the branch to which the Pull Request will target. PR is not created if the flag is not provided. +-w Open the Pull Request on the web browser if it cannot be automatically merged. Requires -b flag. +``` + +=== Example + +``` +./pipeline_generator.sh -c ./templates/aks/aks-pipeline.cfg -n aks-provisioning -d C:/Users/$USERNAME/Desktop/quarkus-project --resource-group devonfw --storage-account hangar --storage-container aks-state -b develop -w +``` + +=== Appendix: Interacting with the cluster + +NOTE: Make sure you have https://kubernetes.io/docs/tasks/tools/#kubectl[kubectl] installed. + +In order to interact with your cluster you will need to download the artifact `kubeconfig` generated by the cluster provisioning pipeline on the location it is expected by default (`~/.kube/config`) or either: + +``` +# via environment variable (you can add this on your profile) +export KUBECONFIG= +kubectl + +# via command-line flag +kubectl --kubeconfig= +``` diff --git a/documentation/src/azure-devops/setup-aks-provisioning-pipeline.asciidoc b/documentation/src/azure-devops/setup-aks-provisioning-pipeline.asciidoc new file mode 100644 index 000000000..a6d00e918 --- /dev/null +++ b/documentation/src/azure-devops/setup-aks-provisioning-pipeline.asciidoc @@ -0,0 +1,7 @@ +:provider: Azure Devops +:pipeline_type: pipeline +:trigger_sentence_azure: +:pipeline_type2: pipeline +:path_provider: azure-devops +:aks_variables_path: Azure DevOps > Pipelines > Library > `aks-variables` +include::../common_templates/setup-aks-provisioning-pipeline.asciidoc[] \ No newline at end of file diff --git a/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc b/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc new file mode 100644 index 000000000..f43cd4d37 --- /dev/null +++ b/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc @@ -0,0 +1,112 @@ += Setting up a Azure AKS provisioning {pipeline_type} on {provider} + +In this section we will create a {pipeline_type} which will provision an Azure AKS cluster. This {pipeline_type} will be configured to be manually triggered by the user. As part of AKS cluster provisioning, a NGINX Ingress controller is deployed and a variable group with the name `aks-variables` is created, which contains, among others, the DNS name of the Ingress controller, that you you will need to add as CNAME record on the domains used in your application Ingress manifest files. Refer to the appendix for more details. + +The creation of the {pipeline_type} will follow the project workflow, so a new branch named `feature/aks-provisioning` will be created, the YAML file for the {pipeline_type} and the terraform files for creating the cluster will be pushed to it. + +Then, a Pull Request (PR) will be created in order to merge the new branch into the appropiate branch (provided in `-b` flag). The PR will be automatically merged if the repository policies are met. If the merge is not possible, either the PR URL will be shown as output, or it will be opened in your web browser if using `-w` flag. + +The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` will automatically create this new branch, create the AKS provisioning {pipeline_type} based on the YAML template, create the Pull Request and, if it is possible, merge this new branch into the specified branch. + +== Prerequisites + +ifdef::trigger_sentence_azure[ * Install the https://marketplace.visualstudio.com/items?itemName=ms-devlabs.custom-terraform-tasks[Terraform extension] for Azure DevOps.] +ifdef::trigger_sentence_azure[ * Create a https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#create-a-service-connection[service connection] to Azure Resource Manager and name it `aks-connection`. If you already have a service connection available or you need a specific connection name, please update `aks-pipeline.cfg` accordingly.] + +ifdef::trigger_sentence_gitlab[ * Add AZURE credentials as [Variables] in your repository and name it `AZURE_USERNAME`, `AZURE_PASSWORD`. If you already have a available credentials or you need a specific credentials connection, please update `aks-provisioning.yml` accordingly.] + +* An Azure resource group in the desired cluster location (e.g. `westeurope`). You can use an existing one or create a new one with the following command: + +``` +az group create -n -l +``` + +* An Azure storage account within the previous resource group. You can use an existing one or create a new one with the following command: + +``` +az storage account create -n -g -l +``` + +* An Azure storage container in Azure within the previous storage account. You can use an existing one or create a new one with the following command: + +``` +az storage container create -n --account-name +``` + +* This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled the latest changes with `git pull`). + +== Creating the {pipeline_type} using provided script + +Before executing the script you will need to customize some input variables about the environment. To do so, you can either edit `terraform.tfvars` file or take advantage of the `set-terraform-variables.sh` script located at `/scripts/environment-provisioning/azure/aks`, which allows you to create or update values for the required variables, passing them as flags. As a full example: + +``` +./set-terraform-variables.sh --location --resource_group_name --instance_type --worker_node_count --dns_prefix +``` + +=== Usage +``` +pipeline_generator.sh \ + -c \ + -n \ + -d \ + --cluster-name \ + --resource-group \ + --storage-account \ + --storage-container \ + [--rancher] \ + [-b ] \ + [-w] +``` + +NOTE: The config file for the AKS provisioning {pipeline_type} is located at `/scripts/pipelines/{path_provider}/templates/aks/aks-pipeline.cfg`. + +=== Flags +``` +-c, --config-file [Required] Configuration file containing pipeline definition. +-n, --pipeline-name [Required] Name that will be set to the pipeline. +-d, --local-directory [Required] Local directory of your project (the path should always be using '/' and not '\'). + --cluster-name [Required] Name for the cluster. +ifdef::trigger_sentence_azure[ --resource-group [Required] Name of the resource group for the cluster. ] +ifdef::trigger_sentence_azure[ --storage--account [Required] Name of the storage account for the cluster. ] + --storage-container [Required] Name of the storage container where the Terraform state of the cluster will be stored. + --rancher Install Rancher to manage the cluster. +-b, --target-branch Name of the branch to which the Pull Request will target. PR is not created if the flag is not provided. +-w Open the Pull Request on the web browser if it cannot be automatically merged. Requires -b flag. +``` + +=== Example + +``` +ifdef::trigger_sentence_azure[ ./pipeline_generator.sh -c ./templates/aks/aks-pipeline.cfg -n aks-provisioning -d C:/Users/$USERNAME/Desktop/quarkus-project --cluster-name devon-hangar --resource-group devonfw --storage-account hangar --storage-container aks-state --rancher -b develop -w ] + +ifdef::trigger_sentence_gitlab[ ./pipeline_generator.sh -c ./templates/aks/aks-pipeline.cfg -n aks-provisioning -d C:/Users/$USERNAME/Desktop/quarkus-project --cluster-name devon-hangar --storage-container aks-state --rancher -b develop -w ] +``` + +NOTE: Rancher is installed on the cluster after provisioning when using the above command. + +=== Appendix: Interacting with the cluster + +NOTE: Make sure you have https://kubernetes.io/docs/tasks/tools/#kubectl[kubectl] installed. + +In order to interact with your cluster you will need to download the artifact `kubeconfig` generated by the cluster provisioning {pipeline_type} on the location it is expected by default (`~/.kube/config`) or either: + +``` +# via environment variable (you can add this on your profile) +export KUBECONFIG= +kubectl + +# via command-line flag +kubectl --kubeconfig= +``` + +To get the DNS name of the NGINX Ingress controller on the AKS cluster, go into {aks_variables_path}. + +Rancher, if installed, will be available on `https:///dashboard`. You will be asked for an initial password, which can be retrieved with: + +``` +kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}{{"\n"}}' +``` + +=== Appendix: Destroying the cluster + +To destroy the provisioned resources, set `operation` {pipeline_type} variable value to `destroy` and run the {pipeline_type}. \ No newline at end of file diff --git a/documentation/src/gitlab/setup-aks-provisioning-pipeline.asciidoc b/documentation/src/gitlab/setup-aks-provisioning-pipeline.asciidoc new file mode 100644 index 000000000..d2a70a50b --- /dev/null +++ b/documentation/src/gitlab/setup-aks-provisioning-pipeline.asciidoc @@ -0,0 +1,7 @@ +:provider: Gitlab +:pipeline_type: pipeline +:trigger_sentence_gitlab: +:pipeline_type2: gitlab +:path_provider: gitlab +:aks_variables_path: Group > ci_cd settings > Variables (devon-hangar > ci_cd settings > Variables) +include::../common_templates/setup-aks-provisioning-pipeline.asciidoc[] \ No newline at end of file diff --git a/scripts/pipelines/gitlab/pipeline_generator.sh b/scripts/pipelines/gitlab/pipeline_generator.sh index 3f04b4d29..2ca9bcb0d 100644 --- a/scripts/pipelines/gitlab/pipeline_generator.sh +++ b/scripts/pipelines/gitlab/pipeline_generator.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e -FLAGS=$(getopt -a --options c:n:d:a:b:l:i:u:p:hw --long "config-file:,pipeline-name:,local-directory:,artifact-path:,target-branch:,language:,build-pipeline-name:,sonar-url:,sonar-token:,image-name:,registry-user:,registry-password:,resource-group:,storage-account:,storage-container:,cluster-name:,s3-bucket:,s3-key-path:,quality-pipeline-name:,dockerfile:,test-pipeline-name:,aws-access-key:,aws-secret-access-key:,aws-region:,help" -- "$@") +FLAGS=$(getopt -a --options c:n:d:a:b:l:i:u:p:hw --long "config-file:,pipeline-name:,local-directory:,artifact-path:,target-branch:,language:,build-pipeline-name:,sonar-url:,sonar-token:,image-name:,registry-user:,registry-password:,resource-group:,storage-account:,storage-container:,cluster-name:,s3-bucket:,s3-key-path:,quality-pipeline-name:,dockerfile:,test-pipeline-name:,aws-access-key:,aws-secret-access-key:,aws-region:,help,rancher" -- "$@") eval set -- "$FLAGS" while true; do @@ -20,6 +20,7 @@ while true; do --resource-group) resourceGroupName=$2; shift 2;; --storage-account) storageAccountName=$2; shift 2;; --storage-container) storageContainerName=$2; shift 2;; + --rancher) installRancher="true"; shift 1;; --cluster-name) clusterName=$2; shift 2;; --s3-bucket) s3Bucket=$2; shift 2;; --s3-key-path) s3KeyPath=$2; shift 2;; diff --git a/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg b/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg index ef9c0c861..3f9f0084a 100644 --- a/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg +++ b/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg @@ -1,11 +1,11 @@ # Mandatory flags. mandatoryFalgs="$pipelineName,$localDirectory,$clusterName,$storageContainerName," #$resourceGroupName,$storageAccountName,$storageContainerName," # Path to the templates. -templatesPath="scripts/pipelines/github/templates/aks" +templatesPath="scripts/pipelines/gitlab/templates/aks" #Path to common kubernetes templates. -commonKubernetesPath="scripts/pipelines/github/templates/common/kubernetes" +commonKubernetesPath="scripts/pipelines/gitlab/templates/common/kubernetes" # aks-provision YAML file name. -yamlFile=".gitlab-ci.yml" +yamlFile="aks-provisioning.yml" # Source branch. sourceBranch="feature/aks-provisioning" # Path to terraform templates. @@ -53,8 +53,8 @@ function addPipelineVariables { export clusterName # export resourceGroupName # export storageAccountName - # export storageContainerName + export storageContainerName export installRancher export operation - specificEnvSubstList='${clusterName} ${installRancher} ${operation}' #${resourceGroupName} ${storageAccountName} ${storageContainerName} + specificEnvSubstList='${clusterName} ${operation} ${storageContainerName} ${installRancher}' #${resourceGroupName} ${storageAccountName} } diff --git a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template index 4e18313d4..86edb9fce 100644 --- a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template +++ b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template @@ -13,17 +13,17 @@ workflow: variables: CLUSTER_NAME: - value: $clusterName + value: "$clusterName" description: "Name for the AWS EKS cluster to be created" OPERATION: - value: $operation + value: "$operation" description: "Operation to perform on cluster. Create or Destroy." INSTALL_RANCHER: - value: $installRancher + value: "$installRancher" description: "Installs Rancher on AKS when set to true." #CLUSTER_NAME: devon-hangar #OPERATION: create - TF_STATE_NAME: $storageContainerName #"TF_AKS_STATE_FILE" + TF_STATE_NAME: "$storageContainerName" #"TF_AKS_STATE_FILE" TF_CACHE_KEY: default TF_ROOT: "${CI_PROJECT_DIR}/azure/aks" TF_USERNAME: ${GITLAB_USER_NAME} @@ -156,7 +156,7 @@ Install-rancher: - chmod 755 install-rancher.sh - ./install-rancher.sh ${aks_dns_name} rules: - - if: '$INSTALL_RANCHER == "true"' + - if: '$INSTALL_RANCHER == "true" && $OPERATION == "create"' when: always Destroy-terraform: diff --git a/scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml b/scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml index 34904840a..06d54fb5b 100644 --- a/scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml +++ b/scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml @@ -1,12 +1,12 @@ include: - '.pipelines/*.yml' -stages: - - build - - test - - quality - - package +# stages: +# - build +# - test +# - quality +# - package -default: - image: maven:3-jdk-11 - tags: ['docker_ruby'] \ No newline at end of file +# default: +# image: maven:3-jdk-11 +# tags: ['docker_ruby'] \ No newline at end of file From 13a55e26922b04f6b1802c9909d1dea09725c33c Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Thu, 8 Sep 2022 11:43:56 +0530 Subject: [PATCH 10/35] updated --- .../pipelines/common/pipeline_generator.lib | 334 +++++++++--------- 1 file changed, 167 insertions(+), 167 deletions(-) diff --git a/scripts/pipelines/common/pipeline_generator.lib b/scripts/pipelines/common/pipeline_generator.lib index 9061d8152..a59347415 100644 --- a/scripts/pipelines/common/pipeline_generator.lib +++ b/scripts/pipelines/common/pipeline_generator.lib @@ -1,167 +1,167 @@ -#!/bin/bash -function help { - echo "" - echo "Generates a pipeline on $provider based on the given definition." - echo "" - echo "Common flags:" - echo " -c, --config-file [Required] Configuration file containing pipeline definition." - echo " -n, --pipeline-name [Required] Name that will be set to the pipeline." - echo " -d, --local-directory [Required] Local directory of your project." - echo " -a, --artifact-path Path to be persisted as an artifact after pipeline execution, e.g. where the application stores logs or any other blob on runtime." - echo " -b, --target-branch Name of the branch to which the Pull Request will target. PR is not created if the flag is not provided." - echo " -w Open the Pull Request on the web browser if it cannot be automatically merged. Requires -b flag." - echo "" - echo "Build pipeline flags:" - echo " -l, --language [Required] Language or framework of the project." - echo " -t, --target-directory Target directory of build process. Takes precedence over the language/framework default one." - echo "" - echo "Test pipeline flags:" - echo " -l, --language [Required] Language or framework of the project." - [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build pipeline name." - echo "" - echo "Quality pipeline flags:" - echo " -l, --language [Required] Language or framework of the project." - echo " --sonar-url [Required] Sonarqube URL." - echo " --sonar-token [Required] Sonarqube token." - [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build pipeline name." - [ "$provider" == "azure-devops" ] && echo " --test-pipeline-name [Required] Test pipeline name." - echo "" - echo "Package pipeline flags:" - echo " -l, --language [Required, if dockerfile not set] Language or framework of the project." - echo " --dockerfile [Required, if language not set] Path from the root of the project to its Dockerfile. Takes precedence over the language/framework default one." - [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build pipeline name." - echo " --quality-pipeline-name [Required] Quality pipeline name." - echo " -i, --image-name [Required] Name (excluding tag) for the generated container image." - echo " -u, --registry-user [Required, unless AWS] Container registry login user." - echo " -p, --registry-password [Required, unless AWS] Container registry login password." - echo " --aws-access-key [Required, if AWS] AWS account access key ID. Takes precedence over registry credentials." - echo " --aws-secret-access-key [Required, if AWS] AWS account secret access key." - echo " --aws-region [Required, if AWS] AWS region for ECR." - echo "" - echo "Library package pipeline flags:" - echo " -l, --language [Required] Language or framework of the project." - echo "" - echo "Deploy pipeline flags:" - echo "" - echo "Azure AKS provisioning pipeline flags:" - echo " --resource-group [Required] Name of the resource group for the cluster." - echo " --storage-account [Required] Name of the storage account for the cluster." - echo " --storage-container [Required] Name of the storage container where the Terraform state of the cluster will be stored." - echo "" - echo "AWS EKS provisioning pipeline flags:" - echo " --cluster-name [Required] Name for the cluster." - echo " --s3-bucket [Required] Name of the S3 bucket where the Terraform state of the cluster will be stored." - echo " --s3-key-path [Required] Path within the S3 bucket where the Terraform state of the cluster will be stored." - - exit -} - -function ensurePathFormat { - currentDirectory=$(pwd) - - # When necessary, converts a relative path into an absolute path, and a Windows-style path (e.g. "C:\Users" or C:/Users) into a - # Unix-style path using forward slashes (e.g. "/c/Users"). - localDirectory=${localDirectory//'\'/"/"} - cd "${localDirectory}" || { echo -e "${red}Error: Local directory '${localDirectory}' does not exist. Check provided path (missing quotes?)."; exit 1; } - localDirectory=$(pwd) - - # Return to initial directory - cd "$currentDirectory" -} - -function importConfigFile { - # Import config file. - source $configFile - IFS=, read -ra flags <<< "$mandatoryFlags" - - # Check if the config file was supplied. - if test -z "$configFile" - then - echo -e "${red}Error: Pipeline definition configuration file not specified." >&2 - exit 2 - fi - - # Check if the required flags in the config file have been activated. - for flag in "${flags[@]}" - do - if test -z $flag - then - echo -e "${red}Error: Missing parameters, some flags are mandatory." >&2 - echo -e "${red}Use -h or --help flag to display help." >&2 - exit 2 - fi - done -} - -function checkInstallations { - # Check if Git is installed - if ! [ -x "$(command -v git)" ]; then - echo -e "${red}Error: Git is not installed." >&2 - exit 127 - fi - - # Check if the CLI tool for the provider is installed - if ([ "$provider" == "github" ] && ! [ -x "$(command -v gh)" ]); then - echo -e "${red}Error: Github CLI is not installed." >&2 - exit 127 - elif ([ "$provider" == "azure-devops" ] && ! [ -x "$(command -v az)" ]); then - echo -e "${red}Error: Azure CLI is not installed." >&2 - exit 127 - elif ([ "$provider" == "gitlab" ] && ! [ -x "$(command -v glab)" ]); then - echo -e "${red}Error: Azure CLI is not installed." >&2 - exit 127 - fi - - # Check if Python is installed - if ! [ -x "$(command -v python)" ]; then - echo -e "${red}Error: Python is not installed." >&2 - exit 127 - fi -} - -function createNewBranch { - echo -e "${green}Creating the new branch: ${sourceBranch}..." - echo -ne ${white} - - # Create the new branch. - cd "${localDirectory}" - git checkout -b ${sourceBranch} -} - -function copyYAMLFile { - echo -e "${green}Copying the corresponding files into your directory..." - echo -ne ${white} - - # Create .pipelines and scripts if they do not exist. - mkdir -p "${localDirectory}/$scriptFilePath" - - # Generate pipeline YAML from template and put it in the repository. - # We cannot use a variable in the definition of resource in the pipeline so we have to use a placeholder to replace it with the value we need - commonEnvSubstList='${buildPipelineName} ${testPipelineName} ${qualityPipelineName} ${pipelineName}' - envsubst "${commonEnvSubstList} ${specificEnvSubstList}" < "${hangarPath}/${templatesPath}/${yamlFile}.template" > "${localDirectory}/${pipelinePath}/${yamlFile}" -} - -function copyCommonScript { - echo -e "${green}Copying the script(s) common to any pipeline files into your directory..." - echo -ne ${white} - - ! (ls "${hangarPath}/${commonTemplatesPath}"/*.sh) &> /dev/null || cp "${hangarPath}/${commonTemplatesPath}"/*.sh "${localDirectory}/${scriptFilePath}" -} - -function commitCommonFiles { - echo -e "${green}Commiting and pushing into Git remote..." - echo -ne ${white} - - # Move into the project's directory and pushing the template into the Azure DevOps repository. - cd ${localDirectory} - - # Add the YAML files. - git add "$pipelinePath" -f - - # Git commit and push it into the repository. - # changing all files to be executable - find "$pipelinePath" -type f -name '*.sh' -exec git update-index --chmod=+x {} \; - - git commit -m "Adding the source YAML" - git push -u origin ${sourceBranch} -} +#!/bin/bash +function help { + echo "" + echo "Generates a pipeline on $provider based on the given definition." + echo "" + echo "Common flags:" + echo " -c, --config-file [Required] Configuration file containing pipeline definition." + echo " -n, --pipeline-name [Required] Name that will be set to the pipeline." + echo " -d, --local-directory [Required] Local directory of your project." + echo " -a, --artifact-path Path to be persisted as an artifact after pipeline execution, e.g. where the application stores logs or any other blob on runtime." + echo " -b, --target-branch Name of the branch to which the Pull Request will target. PR is not created if the flag is not provided." + echo " -w Open the Pull Request on the web browser if it cannot be automatically merged. Requires -b flag." + echo "" + echo "Build pipeline flags:" + echo " -l, --language [Required] Language or framework of the project." + echo " -t, --target-directory Target directory of build process. Takes precedence over the language/framework default one." + echo "" + echo "Test pipeline flags:" + echo " -l, --language [Required] Language or framework of the project." + [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build pipeline name." + echo "" + echo "Quality pipeline flags:" + echo " -l, --language [Required] Language or framework of the project." + echo " --sonar-url [Required] Sonarqube URL." + echo " --sonar-token [Required] Sonarqube token." + [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build pipeline name." + [ "$provider" == "azure-devops" ] && echo " --test-pipeline-name [Required] Test pipeline name." + echo "" + echo "Package pipeline flags:" + echo " -l, --language [Required, if dockerfile not set] Language or framework of the project." + echo " --dockerfile [Required, if language not set] Path from the root of the project to its Dockerfile. Takes precedence over the language/framework default one." + [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build pipeline name." + echo " --quality-pipeline-name [Required] Quality pipeline name." + echo " -i, --image-name [Required] Name (excluding tag) for the generated container image." + echo " -u, --registry-user [Required, unless AWS] Container registry login user." + echo " -p, --registry-password [Required, unless AWS] Container registry login password." + echo " --aws-access-key [Required, if AWS] AWS account access key ID. Takes precedence over registry credentials." + echo " --aws-secret-access-key [Required, if AWS] AWS account secret access key." + echo " --aws-region [Required, if AWS] AWS region for ECR." + echo "" + echo "Library package pipeline flags:" + echo " -l, --language [Required] Language or framework of the project." + echo "" + echo "Deploy pipeline flags:" + echo "" + echo "Azure AKS provisioning pipeline flags:" + echo " --resource-group [Required] Name of the resource group for the cluster." + echo " --storage-account [Required] Name of the storage account for the cluster." + echo " --storage-container [Required] Name of the storage container where the Terraform state of the cluster will be stored." + echo "" + echo "AWS EKS provisioning pipeline flags:" + echo " --cluster-name [Required] Name for the cluster." + echo " --s3-bucket [Required] Name of the S3 bucket where the Terraform state of the cluster will be stored." + echo " --s3-key-path [Required] Path within the S3 bucket where the Terraform state of the cluster will be stored." + + exit +} + +function ensurePathFormat { + currentDirectory=$(pwd) + + # When necessary, converts a relative path into an absolute path, and a Windows-style path (e.g. "C:\Users" or C:/Users) into a + # Unix-style path using forward slashes (e.g. "/c/Users"). + localDirectory=${localDirectory//'\'/"/"} + cd "${localDirectory}" || { echo -e "${red}Error: Local directory '${localDirectory}' does not exist. Check provided path (missing quotes?)."; exit 1; } + localDirectory=$(pwd) + + # Return to initial directory + cd "$currentDirectory" +} + +function importConfigFile { + # Import config file. + source $configFile + IFS=, read -ra flags <<< "$mandatoryFlags" + + # Check if the config file was supplied. + if test -z "$configFile" + then + echo -e "${red}Error: Pipeline definition configuration file not specified." >&2 + exit 2 + fi + + # Check if the required flags in the config file have been activated. + for flag in "${flags[@]}" + do + if test -z $flag + then + echo -e "${red}Error: Missing parameters, some flags are mandatory." >&2 + echo -e "${red}Use -h or --help flag to display help." >&2 + exit 2 + fi + done +} + +function checkInstallations { + # Check if Git is installed + if ! [ -x "$(command -v git)" ]; then + echo -e "${red}Error: Git is not installed." >&2 + exit 127 + fi + + # Check if the CLI tool for the provider is installed + if ([ "$provider" == "github" ] && ! [ -x "$(command -v gh)" ]); then + echo -e "${red}Error: Github CLI is not installed." >&2 + exit 127 + elif ([ "$provider" == "azure-devops" ] && ! [ -x "$(command -v az)" ]); then + echo -e "${red}Error: Azure CLI is not installed." >&2 + exit 127 + elif ([ "$provider" == "gitlab" ] && ! [ -x "$(command -v glab)" ]); then + echo -e "${red}Error: Azure CLI is not installed." >&2 + exit 127 + fi + + # Check if Python is installed + if ! [ -x "$(command -v python)" ]; then + echo -e "${red}Error: Python is not installed." >&2 + exit 127 + fi +} + +function createNewBranch { + echo -e "${green}Creating the new branch: ${sourceBranch}..." + echo -ne ${white} + + # Create the new branch. + cd "${localDirectory}" + git checkout -b ${sourceBranch} +} + +function copyYAMLFile { + echo -e "${green}Copying the corresponding files into your directory..." + echo -ne ${white} + + # Create .pipelines and scripts if they do not exist. + mkdir -p "${localDirectory}/$scriptFilePath" + + # Generate pipeline YAML from template and put it in the repository. + # We cannot use a variable in the definition of resource in the pipeline so we have to use a placeholder to replace it with the value we need + commonEnvSubstList='${buildPipelineName} ${testPipelineName} ${qualityPipelineName} ${pipelineName}' + envsubst "${commonEnvSubstList} ${specificEnvSubstList}" < "${hangarPath}/${templatesPath}/${yamlFile}.template" > "${localDirectory}/${pipelinePath}/${yamlFile}" +} + +function copyCommonScript { + echo -e "${green}Copying the script(s) common to any pipeline files into your directory..." + echo -ne ${white} + + ! (ls "${hangarPath}/${commonTemplatesPath}"/*.sh) &> /dev/null || cp "${hangarPath}/${commonTemplatesPath}"/*.sh "${localDirectory}/${scriptFilePath}" +} + +function commitCommonFiles { + echo -e "${green}Commiting and pushing into Git remote..." + echo -ne ${white} + + # Move into the project's directory and pushing the template into the Azure DevOps repository. + cd ${localDirectory} + + # Add the YAML files. + git add "$pipelinePath" -f + + # Git commit and push it into the repository. + # changing all files to be executable + find "$pipelinePath" -type f -name '*.sh' -exec git update-index --chmod=+x {} \; + + git commit -m "Adding the source YAML" + git push -u origin ${sourceBranch} +} From f17540fb7e425c0f3bfba350dfe44b774eebdecb Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Thu, 8 Sep 2022 11:49:16 +0530 Subject: [PATCH 11/35] updated --- .../gitlab/templates/aks/aks-pipeline.cfg | 120 +++--- .../aks/aks-provisioning.yml.template | 346 +++++++++--------- .../gitlab/templates/aks/obtain-dns.sh | 42 +-- 3 files changed, 254 insertions(+), 254 deletions(-) diff --git a/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg b/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg index 3f9f0084a..956ed68af 100644 --- a/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg +++ b/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg @@ -1,60 +1,60 @@ -# Mandatory flags. -mandatoryFalgs="$pipelineName,$localDirectory,$clusterName,$storageContainerName," #$resourceGroupName,$storageAccountName,$storageContainerName," -# Path to the templates. -templatesPath="scripts/pipelines/gitlab/templates/aks" -#Path to common kubernetes templates. -commonKubernetesPath="scripts/pipelines/gitlab/templates/common/kubernetes" -# aks-provision YAML file name. -yamlFile="aks-provisioning.yml" -# Source branch. -sourceBranch="feature/aks-provisioning" -# Path to terraform templates. -terraformTemplatesPath="scripts/environment-provisioning/azure/aks" -# Path to terraform scripts. -terraformPath=".terraform/aks" -# Default cluster operation. -operation="create" -# Install Rancher on AKS cluster. -if test -z $installRancher -then - installRancher=false -fi - -# Function that copies the necessary scripts into the directory. -function copyScript { - # Create .terraform/aks folder if it does not exist. - mkdir -p "${localDirectory}/${terraformPath}" - - # Copy the terraform files. - cd "${hangarPath}/${terraformTemplatesPath}" - cp * "${localDirectory}/${terraformPath}" - - # Copy the script for the DNS name into the directory. - cp "${hangarPath}/${templatesPath}/obtain-dns.sh" "${localDirectory}/${scriptFilePath}/obtain-dns.sh" - - # Copy the common files for kubernetes - cp "${hangarPath}/${commonKubernetesPath}"/*.sh "${localDirectory}/${scriptFilePath}" -} - -function commitFiles { - # Add the terraform files. - git add .terraform -f - - # Changing all files to be executable. - find .terraform -type f -name '*.sh' -exec git update-index --chmod=+x {} \; - - # Git commit and push it into the repository. - git commit -m "Adding the terraform files" - git push -u origin ${sourceBranch} -} - -# Function that adds the variables to be used in the pipeline. -function addPipelineVariables { - export clusterName - # export resourceGroupName - # export storageAccountName - export storageContainerName - export installRancher - export operation - specificEnvSubstList='${clusterName} ${operation} ${storageContainerName} ${installRancher}' #${resourceGroupName} ${storageAccountName} -} +# Mandatory flags. +mandatoryFalgs="$pipelineName,$localDirectory,$clusterName,$storageContainerName," #$resourceGroupName,$storageAccountName,$storageContainerName," +# Path to the templates. +templatesPath="scripts/pipelines/gitlab/templates/aks" +#Path to common kubernetes templates. +commonKubernetesPath="scripts/pipelines/gitlab/templates/common/kubernetes" +# aks-provision YAML file name. +yamlFile="aks-provisioning.yml" +# Source branch. +sourceBranch="feature/aks-provisioning" +# Path to terraform templates. +terraformTemplatesPath="scripts/environment-provisioning/azure/aks" +# Path to terraform scripts. +terraformPath=".terraform/aks" +# Default cluster operation. +operation="create" +# Install Rancher on AKS cluster. +if test -z $installRancher +then + installRancher=false +fi + +# Function that copies the necessary scripts into the directory. +function copyScript { + # Create .terraform/aks folder if it does not exist. + mkdir -p "${localDirectory}/${terraformPath}" + + # Copy the terraform files. + cd "${hangarPath}/${terraformTemplatesPath}" + cp * "${localDirectory}/${terraformPath}" + + # Copy the script for the DNS name into the directory. + cp "${hangarPath}/${templatesPath}/obtain-dns.sh" "${localDirectory}/${scriptFilePath}/obtain-dns.sh" + + # Copy the common files for kubernetes + cp "${hangarPath}/${commonKubernetesPath}"/*.sh "${localDirectory}/${scriptFilePath}" +} + +function commitFiles { + # Add the terraform files. + git add .terraform -f + + # Changing all files to be executable. + find .terraform -type f -name '*.sh' -exec git update-index --chmod=+x {} \; + + # Git commit and push it into the repository. + git commit -m "Adding the terraform files" + git push -u origin ${sourceBranch} +} + +# Function that adds the variables to be used in the pipeline. +function addPipelineVariables { + export clusterName + # export resourceGroupName + # export storageAccountName + export storageContainerName + export installRancher + export operation + specificEnvSubstList='${clusterName} ${operation} ${storageContainerName} ${installRancher}' #${resourceGroupName} ${storageAccountName} +} diff --git a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template index 86edb9fce..d27428a11 100644 --- a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template +++ b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template @@ -1,173 +1,173 @@ -default: - image: - name: ubuntu:latest - entrypoint: - - /usr/bin/env - - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - -workflow: - rules: - - if: '$CI_PIPELINE_SOURCE == "web"' - when: always - - when: never - -variables: - CLUSTER_NAME: - value: "$clusterName" - description: "Name for the AWS EKS cluster to be created" - OPERATION: - value: "$operation" - description: "Operation to perform on cluster. Create or Destroy." - INSTALL_RANCHER: - value: "$installRancher" - description: "Installs Rancher on AKS when set to true." - #CLUSTER_NAME: devon-hangar - #OPERATION: create - TF_STATE_NAME: "$storageContainerName" #"TF_AKS_STATE_FILE" - TF_CACHE_KEY: default - TF_ROOT: "${CI_PROJECT_DIR}/azure/aks" - TF_USERNAME: ${GITLAB_USER_NAME} - TF_PASSWORD: ${GITLAB_PAT} - TF_ADDRESS: "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/terraform/state/${TF_STATE_NAME}" - TF_HTTP_ADDRESS: ${TF_ADDRESS} - TF_HTTP_LOCK_ADDRESS: ${TF_ADDRESS}/lock - TF_HTTP_LOCK_METHOD: POST - TF_HTTP_UNLOCK_ADDRESS: ${TF_ADDRESS}/lock - TF_HTTP_UNLOCK_METHOD: DELETE - TF_HTTP_USERNAME: ${TF_USERNAME} - TF_HTTP_PASSWORD: ${TF_PASSWORD} - TF_HTTP_RETRY_WAIT_MIN: 5 - -.Prerequisites_install: &Prerequisites_install - before_script: - - apt-get update - - apt-get install sudo -y - - apt-get install curl -y - - apt-get install zip -y - - apt-get install -y wget - -.configure_azcli: &login_azcli - # INSTALL AZCLI - - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - - az --version - - az login #--username "${AZURE_USERNAME}" --password "${AZURE_PASSWORD}" - -.packages: &configure_packages - # INSTALL KUBECTL - - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - - chmod +x ./kubectl - - mv ./kubectl /usr/local/bin/kubectl - # INSTALL HELM - - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - - chmod +x get_helm.sh - - DESIRED_VERSION=v3.9.0 ./get_helm.sh - -.install-terraform: &install-terraform - # INSTALL - - wget -nv https://releases.hashicorp.com/terraform/1.2.6/terraform_1.2.6_linux_amd64.zip - - unzip -qq terraform_1.2.6_linux_amd64.zip - - sudo mv terraform /usr/local/bin - -.configure_kubeconfig: &configure_kubeconfig - # - "curl 'https://gitlab.com/$CI_PROJECT_PATH/-/jobs/$CI_JOB_ID/artifacts/${CI_COMMIT_REF_NAME}/raw/kubeconfig?job=$CI_JOB_STAGE&JOB-TOKEN=$CI_JOB_TOKEN'" - - export KUBECONFIG=${KUBECONFIG} - -Provision: - <<: *Prerequisites_install - script: - - echo "hello $CLUSTER_NAME:$OPERATION, $TF_STATE_NAME" - - *install-terraform - - mkdir -p ${CI_PROJECT_DIR}/scripts - - cd ${CI_PROJECT_DIR}/scripts - - *login_azcli - - cd .. - - terraform --version - - cd ${TF_ROOT} - - terraform init -var cluster_name=${CLUSTER_NAME} - - terraform apply -var cluster_name=${CLUSTER_NAME} --auto-approve - - mv ${TF_ROOT}/kubeconfig ${CI_PROJECT_DIR} - - curl -s https://raw.githubusercontent.com/profclems/glab/trunk/scripts/install.sh | sudo sh - - glab auth login --token $GITLAB_PAT - - cat ${CI_PROJECT_DIR}/kubeconfig | glab variable set -g devon-hangar KUBECONFIG -t "file" - artifacts: - paths: - - "./kubeconfig" - cache: - key: "scripts" - paths: - - "${CI_PROJECT_DIR}/scripts" - rules: - - if: '$OPERATION == "create"' - when: always - -Install_nginx: - <<: *Prerequisites_install - needs: [Provision] - cache: - key: "scripts" - paths: - - "${CI_PROJECT_DIR}/scripts" - script: - - cd ${CI_PROJECT_DIR}/scripts - - *configure_packages - - *configure_kubeconfig - - chmod 755 install-nginx-ingress.sh - - ./install-nginx-ingress.sh - rules: - - if: '$OPERATION == "create"' - when: always - -Obtain_dns: - <<: *Prerequisites_install - script: - - cd ${CI_PROJECT_DIR}/scripts - - *configure_packages - - *login_azcli - - *configure_kubeconfig - # Obtain-dns - - chmod 755 obtain-dns.sh - - ./obtain-dns.sh ${CLUSTER_NAME} - - dnsname="${CLUSTER_NAME}.westeurope.cloudapp.azure.com" - # Create aks_dns_name variable - - curl -s https://raw.githubusercontent.com/profclems/glab/trunk/scripts/install.sh | sudo sh - - glab auth login --token $GITLAB_PAT - - glab variable set -g devon-hangar aks_dns_name -v "$dnsname" -t "env_var" - rules: - - if: '$OPERATION == "create"' - when: always - needs: - - Provision - #- Install_nginx - -Install-rancher: - <<: *Prerequisites_install - needs: - - Provision - - Obtain_dns - cache: - key: "scripts" - paths: - - "${CI_PROJECT_DIR}/scripts" - script: - - cd ${CI_PROJECT_DIR}/scripts - - *configure_packages - - *configure_kubeconfig - # INSTALL RANCHER - - chmod 755 install-rancher.sh - - ./install-rancher.sh ${aks_dns_name} - rules: - - if: '$INSTALL_RANCHER == "true" && $OPERATION == "create"' - when: always - -Destroy-terraform: - <<: *Prerequisites_install - script: - - *install-terraform - - *configure_packages - - cd ${TF_ROOT} - - ls -lrta - - terraform init - - terraform apply -destroy -var cluster_name=${CLUSTER_NAME} --auto-approve - rules: - - if: '$OPERATION == "destroy"' - when: always +default: + image: + name: ubuntu:latest + entrypoint: + - /usr/bin/env + - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +workflow: + rules: + - if: '$CI_PIPELINE_SOURCE == "web"' + when: always + - when: never + +variables: + CLUSTER_NAME: + value: "$clusterName" + description: "Name for the AWS EKS cluster to be created" + OPERATION: + value: "$operation" + description: "Operation to perform on cluster. Create or Destroy." + INSTALL_RANCHER: + value: "$installRancher" + description: "Installs Rancher on AKS when set to true." + #CLUSTER_NAME: devon-hangar + #OPERATION: create + TF_STATE_NAME: "$storageContainerName" #"TF_AKS_STATE_FILE" + TF_CACHE_KEY: default + TF_ROOT: "${CI_PROJECT_DIR}/azure/aks" + TF_USERNAME: ${GITLAB_USER_NAME} + TF_PASSWORD: ${GITLAB_PAT} + TF_ADDRESS: "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/terraform/state/${TF_STATE_NAME}" + TF_HTTP_ADDRESS: ${TF_ADDRESS} + TF_HTTP_LOCK_ADDRESS: ${TF_ADDRESS}/lock + TF_HTTP_LOCK_METHOD: POST + TF_HTTP_UNLOCK_ADDRESS: ${TF_ADDRESS}/lock + TF_HTTP_UNLOCK_METHOD: DELETE + TF_HTTP_USERNAME: ${TF_USERNAME} + TF_HTTP_PASSWORD: ${TF_PASSWORD} + TF_HTTP_RETRY_WAIT_MIN: 5 + +.Prerequisites_install: &Prerequisites_install + before_script: + - apt-get update + - apt-get install sudo -y + - apt-get install curl -y + - apt-get install zip -y + - apt-get install -y wget + +.configure_azcli: &login_azcli + # INSTALL AZCLI + - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + - az --version + - az login #--username "${AZURE_USERNAME}" --password "${AZURE_PASSWORD}" + +.packages: &configure_packages + # INSTALL KUBECTL + - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + - chmod +x ./kubectl + - mv ./kubectl /usr/local/bin/kubectl + # INSTALL HELM + - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + - chmod +x get_helm.sh + - DESIRED_VERSION=v3.9.0 ./get_helm.sh + +.install-terraform: &install-terraform + # INSTALL + - wget -nv https://releases.hashicorp.com/terraform/1.2.6/terraform_1.2.6_linux_amd64.zip + - unzip -qq terraform_1.2.6_linux_amd64.zip + - sudo mv terraform /usr/local/bin + +.configure_kubeconfig: &configure_kubeconfig + # - "curl 'https://gitlab.com/$CI_PROJECT_PATH/-/jobs/$CI_JOB_ID/artifacts/${CI_COMMIT_REF_NAME}/raw/kubeconfig?job=$CI_JOB_STAGE&JOB-TOKEN=$CI_JOB_TOKEN'" + - export KUBECONFIG=${KUBECONFIG} + +Provision: + <<: *Prerequisites_install + script: + - echo "hello $CLUSTER_NAME:$OPERATION, $TF_STATE_NAME" + - *install-terraform + - mkdir -p ${CI_PROJECT_DIR}/scripts + - cd ${CI_PROJECT_DIR}/scripts + - *login_azcli + - cd .. + - terraform --version + - cd ${TF_ROOT} + - terraform init -var cluster_name=${CLUSTER_NAME} + - terraform apply -var cluster_name=${CLUSTER_NAME} --auto-approve + - mv ${TF_ROOT}/kubeconfig ${CI_PROJECT_DIR} + - curl -s https://raw.githubusercontent.com/profclems/glab/trunk/scripts/install.sh | sudo sh + - glab auth login --token $GITLAB_PAT + - cat ${CI_PROJECT_DIR}/kubeconfig | glab variable set -g devon-hangar KUBECONFIG -t "file" + artifacts: + paths: + - "./kubeconfig" + cache: + key: "scripts" + paths: + - "${CI_PROJECT_DIR}/scripts" + rules: + - if: '$OPERATION == "create"' + when: always + +Install_nginx: + <<: *Prerequisites_install + needs: [Provision] + cache: + key: "scripts" + paths: + - "${CI_PROJECT_DIR}/scripts" + script: + - cd ${CI_PROJECT_DIR}/scripts + - *configure_packages + - *configure_kubeconfig + - chmod 755 install-nginx-ingress.sh + - ./install-nginx-ingress.sh + rules: + - if: '$OPERATION == "create"' + when: always + +Obtain_dns: + <<: *Prerequisites_install + script: + - cd ${CI_PROJECT_DIR}/scripts + - *configure_packages + - *login_azcli + - *configure_kubeconfig + # Obtain-dns + - chmod 755 obtain-dns.sh + - ./obtain-dns.sh ${CLUSTER_NAME} + - dnsname="${CLUSTER_NAME}.westeurope.cloudapp.azure.com" + # Create aks_dns_name variable + - curl -s https://raw.githubusercontent.com/profclems/glab/trunk/scripts/install.sh | sudo sh + - glab auth login --token $GITLAB_PAT + - glab variable set -g devon-hangar aks_dns_name -v "$dnsname" -t "env_var" + rules: + - if: '$OPERATION == "create"' + when: always + needs: + - Provision + #- Install_nginx + +Install-rancher: + <<: *Prerequisites_install + needs: + - Provision + - Obtain_dns + cache: + key: "scripts" + paths: + - "${CI_PROJECT_DIR}/scripts" + script: + - cd ${CI_PROJECT_DIR}/scripts + - *configure_packages + - *configure_kubeconfig + # INSTALL RANCHER + - chmod 755 install-rancher.sh + - ./install-rancher.sh ${aks_dns_name} + rules: + - if: '$INSTALL_RANCHER == "true" && $OPERATION == "create"' + when: always + +Destroy-terraform: + <<: *Prerequisites_install + script: + - *install-terraform + - *configure_packages + - cd ${TF_ROOT} + - ls -lrta + - terraform init + - terraform apply -destroy -var cluster_name=${CLUSTER_NAME} --auto-approve + rules: + - if: '$OPERATION == "destroy"' + when: always diff --git a/scripts/pipelines/gitlab/templates/aks/obtain-dns.sh b/scripts/pipelines/gitlab/templates/aks/obtain-dns.sh index 567cc74a5..c6f431573 100644 --- a/scripts/pipelines/gitlab/templates/aks/obtain-dns.sh +++ b/scripts/pipelines/gitlab/templates/aks/obtain-dns.sh @@ -1,21 +1,21 @@ -#!/bin/bash -ip="$(kubectl get svc nginx-ingress-nginx-ingress-controller --namespace nginx-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" - -while test -z "$ip" -do - sleep 5s - ip="$(kubectl get svc nginx-ingress-nginx-ingress-controller --namespace nginx-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" -done - -# Obtain the AKS cluster name -dnsname=$1 - -ipname=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$ip')].[name]" --output tsv) - -iprg=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$ip')].[resourceGroup]" --output tsv) - -az network public-ip update --resource-group "$iprg" --name "$ipname" --dns-name "$dnsname" - -dns="$(az network public-ip show --resource-group "$iprg" --name "$ipname" --query "[dnsSettings.fqdn]" --output tsv)" - -echo "$dns" +#!/bin/bash +ip="$(kubectl get svc nginx-ingress-nginx-ingress-controller --namespace nginx-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" + +while test -z "$ip" +do + sleep 5s + ip="$(kubectl get svc nginx-ingress-nginx-ingress-controller --namespace nginx-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" +done + +# Obtain the AKS cluster name +dnsname=$1 + +ipname=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$ip')].[name]" --output tsv) + +iprg=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$ip')].[resourceGroup]" --output tsv) + +az network public-ip update --resource-group "$iprg" --name "$ipname" --dns-name "$dnsname" + +dns="$(az network public-ip show --resource-group "$iprg" --name "$ipname" --query "[dnsSettings.fqdn]" --output tsv)" + +echo "$dns" From 200680d77033ada01c2dece0c33781da29692808 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 12 Sep 2022 11:39:45 +0530 Subject: [PATCH 12/35] updated with document --- .../setup-aks-provisioning-pipeline.asciidoc | 2 +- .../setup-aks-provisioning-pipeline.asciidoc | 14 ++-- .../setup-aks-provisioning-pipeline.asciidoc | 2 +- .../gitlab/templates/aks/aks-pipeline.cfg | 4 +- .../aks/aks-provisioning.yml.template | 66 +++++-------------- 5 files changed, 28 insertions(+), 60 deletions(-) diff --git a/documentation/src/azure-devops/setup-aks-provisioning-pipeline.asciidoc b/documentation/src/azure-devops/setup-aks-provisioning-pipeline.asciidoc index a6d00e918..5c32723f9 100644 --- a/documentation/src/azure-devops/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/src/azure-devops/setup-aks-provisioning-pipeline.asciidoc @@ -3,5 +3,5 @@ :trigger_sentence_azure: :pipeline_type2: pipeline :path_provider: azure-devops -:aks_variables_path: Azure DevOps > Pipelines > Library > `aks-variables` +:aks_variables_path: Azure DevOps > Pipelines > Library > 'aks-variables include::../common_templates/setup-aks-provisioning-pipeline.asciidoc[] \ No newline at end of file diff --git a/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc b/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc index f43cd4d37..c40247101 100644 --- a/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc @@ -47,11 +47,11 @@ Before executing the script you will need to customize some input variables abou ``` pipeline_generator.sh \ -c \ - -n \ +ifdef::trigger_sentence_azure,trigger_sentence_github[ -n \] -d \ --cluster-name \ - --resource-group \ - --storage-account \ +ifdef::trigger_sentence_azure,trigger_sentence_github[ --resource-group \] +ifdef::trigger_sentence_azure,trigger_sentence_github[ --storage-account \] --storage-container \ [--rancher] \ [-b ] \ @@ -63,11 +63,11 @@ NOTE: The config file for the AKS provisioning {pipeline_type} is located at `/s === Flags ``` -c, --config-file [Required] Configuration file containing pipeline definition. --n, --pipeline-name [Required] Name that will be set to the pipeline. +ifdef::trigger_sentence_azure,trigger_sentence_github[-n, --pipeline-name [Required] Name that will be set to the pipeline.] -d, --local-directory [Required] Local directory of your project (the path should always be using '/' and not '\'). --cluster-name [Required] Name for the cluster. -ifdef::trigger_sentence_azure[ --resource-group [Required] Name of the resource group for the cluster. ] -ifdef::trigger_sentence_azure[ --storage--account [Required] Name of the storage account for the cluster. ] +ifdef::trigger_sentence_azure,trigger_sentence_github[ --resource-group [Required] Name of the resource group for the cluster. ] +ifdef::trigger_sentence_azure,trigger_sentence_github[ --storage--account [Required] Name of the storage account for the cluster. ] --storage-container [Required] Name of the storage container where the Terraform state of the cluster will be stored. --rancher Install Rancher to manage the cluster. -b, --target-branch Name of the branch to which the Pull Request will target. PR is not created if the flag is not provided. @@ -77,7 +77,7 @@ ifdef::trigger_sentence_azure[ --storage--account [Required] Name of the st === Example ``` -ifdef::trigger_sentence_azure[ ./pipeline_generator.sh -c ./templates/aks/aks-pipeline.cfg -n aks-provisioning -d C:/Users/$USERNAME/Desktop/quarkus-project --cluster-name devon-hangar --resource-group devonfw --storage-account hangar --storage-container aks-state --rancher -b develop -w ] +ifdef::trigger_sentence_azure,trigger_sentence_github[ ./pipeline_generator.sh -c ./templates/aks/aks-pipeline.cfg -n aks-provisioning -d C:/Users/$USERNAME/Desktop/quarkus-project --cluster-name devon-hangar --resource-group devonfw --storage-account hangar --storage-container aks-state --rancher -b develop -w ] ifdef::trigger_sentence_gitlab[ ./pipeline_generator.sh -c ./templates/aks/aks-pipeline.cfg -n aks-provisioning -d C:/Users/$USERNAME/Desktop/quarkus-project --cluster-name devon-hangar --storage-container aks-state --rancher -b develop -w ] ``` diff --git a/documentation/src/gitlab/setup-aks-provisioning-pipeline.asciidoc b/documentation/src/gitlab/setup-aks-provisioning-pipeline.asciidoc index d2a70a50b..468c977ec 100644 --- a/documentation/src/gitlab/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/src/gitlab/setup-aks-provisioning-pipeline.asciidoc @@ -3,5 +3,5 @@ :trigger_sentence_gitlab: :pipeline_type2: gitlab :path_provider: gitlab -:aks_variables_path: Group > ci_cd settings > Variables (devon-hangar > ci_cd settings > Variables) +:aks_variables_path: Group > settings > ci_cd > Variables include::../common_templates/setup-aks-provisioning-pipeline.asciidoc[] \ No newline at end of file diff --git a/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg b/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg index 956ed68af..1f58834d4 100644 --- a/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg +++ b/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg @@ -1,5 +1,5 @@ # Mandatory flags. -mandatoryFalgs="$pipelineName,$localDirectory,$clusterName,$storageContainerName," #$resourceGroupName,$storageAccountName,$storageContainerName," +mandatoryFalgs="$localDirectory,$clusterName,$storageContainerName," # Path to the templates. templatesPath="scripts/pipelines/gitlab/templates/aks" #Path to common kubernetes templates. @@ -51,8 +51,6 @@ function commitFiles { # Function that adds the variables to be used in the pipeline. function addPipelineVariables { export clusterName - # export resourceGroupName - # export storageAccountName export storageContainerName export installRancher export operation diff --git a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template index d27428a11..46ad50f8d 100644 --- a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template +++ b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template @@ -20,10 +20,8 @@ variables: description: "Operation to perform on cluster. Create or Destroy." INSTALL_RANCHER: value: "$installRancher" - description: "Installs Rancher on AKS when set to true." - #CLUSTER_NAME: devon-hangar - #OPERATION: create - TF_STATE_NAME: "$storageContainerName" #"TF_AKS_STATE_FILE" + description: "Installs Rancher on AKS when set to true." + TF_STATE_NAME: "$storageContainerName" TF_CACHE_KEY: default TF_ROOT: "${CI_PROJECT_DIR}/azure/aks" TF_USERNAME: ${GITLAB_USER_NAME} @@ -68,17 +66,14 @@ variables: - unzip -qq terraform_1.2.6_linux_amd64.zip - sudo mv terraform /usr/local/bin -.configure_kubeconfig: &configure_kubeconfig - # - "curl 'https://gitlab.com/$CI_PROJECT_PATH/-/jobs/$CI_JOB_ID/artifacts/${CI_COMMIT_REF_NAME}/raw/kubeconfig?job=$CI_JOB_STAGE&JOB-TOKEN=$CI_JOB_TOKEN'" - - export KUBECONFIG=${KUBECONFIG} +.configure_glab_auth: &glab_auth + - curl -s https://raw.githubusercontent.com/profclems/glab/trunk/scripts/install.sh | sudo sh + - glab auth login --token $GITLAB_PAT Provision: <<: *Prerequisites_install script: - - echo "hello $CLUSTER_NAME:$OPERATION, $TF_STATE_NAME" - *install-terraform - - mkdir -p ${CI_PROJECT_DIR}/scripts - - cd ${CI_PROJECT_DIR}/scripts - *login_azcli - cd .. - terraform --version @@ -86,16 +81,8 @@ Provision: - terraform init -var cluster_name=${CLUSTER_NAME} - terraform apply -var cluster_name=${CLUSTER_NAME} --auto-approve - mv ${TF_ROOT}/kubeconfig ${CI_PROJECT_DIR} - - curl -s https://raw.githubusercontent.com/profclems/glab/trunk/scripts/install.sh | sudo sh - - glab auth login --token $GITLAB_PAT + - *glab_auth - cat ${CI_PROJECT_DIR}/kubeconfig | glab variable set -g devon-hangar KUBECONFIG -t "file" - artifacts: - paths: - - "./kubeconfig" - cache: - key: "scripts" - paths: - - "${CI_PROJECT_DIR}/scripts" rules: - if: '$OPERATION == "create"' when: always @@ -103,58 +90,42 @@ Provision: Install_nginx: <<: *Prerequisites_install needs: [Provision] - cache: - key: "scripts" - paths: - - "${CI_PROJECT_DIR}/scripts" script: - - cd ${CI_PROJECT_DIR}/scripts - *configure_packages - - *configure_kubeconfig - - chmod 755 install-nginx-ingress.sh - - ./install-nginx-ingress.sh + - export KUBECONFIG=${KUBECONFIG} + - chmod 755 .pipelines/scripts/install-nginx-ingress.sh + - ./.pipelines/scripts/install-nginx-ingress.sh rules: - if: '$OPERATION == "create"' when: always Obtain_dns: <<: *Prerequisites_install + needs: [Install_nginx] script: - - cd ${CI_PROJECT_DIR}/scripts - *configure_packages - *login_azcli - - *configure_kubeconfig + - export KUBECONFIG=${KUBECONFIG} # Obtain-dns - - chmod 755 obtain-dns.sh - - ./obtain-dns.sh ${CLUSTER_NAME} + - chmod 755 .pipelines/scripts/obtain-dns.sh + - ./.pipelines/scripts/obtain-dns.sh ${CLUSTER_NAME} - dnsname="${CLUSTER_NAME}.westeurope.cloudapp.azure.com" # Create aks_dns_name variable - - curl -s https://raw.githubusercontent.com/profclems/glab/trunk/scripts/install.sh | sudo sh - - glab auth login --token $GITLAB_PAT + - *glab_auth - glab variable set -g devon-hangar aks_dns_name -v "$dnsname" -t "env_var" rules: - if: '$OPERATION == "create"' when: always - needs: - - Provision - #- Install_nginx Install-rancher: <<: *Prerequisites_install - needs: - - Provision - - Obtain_dns - cache: - key: "scripts" - paths: - - "${CI_PROJECT_DIR}/scripts" + needs: [Obtain_dns] script: - - cd ${CI_PROJECT_DIR}/scripts - *configure_packages - - *configure_kubeconfig + - export KUBECONFIG=${KUBECONFIG} # INSTALL RANCHER - - chmod 755 install-rancher.sh - - ./install-rancher.sh ${aks_dns_name} + - chmod 755 .pipelines/scripts/install-rancher.sh + - ./.pipelines/scripts/install-rancher.sh ${aks_dns_name} rules: - if: '$INSTALL_RANCHER == "true" && $OPERATION == "create"' when: always @@ -165,7 +136,6 @@ Destroy-terraform: - *install-terraform - *configure_packages - cd ${TF_ROOT} - - ls -lrta - terraform init - terraform apply -destroy -var cluster_name=${CLUSTER_NAME} --auto-approve rules: From c0a821638920d8dc7407d1dacb76631c2ca8fe7b Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 12 Sep 2022 06:11:13 +0000 Subject: [PATCH 13/35] automatic commit dor docs --- .../setup-aks-provisioning-pipeline.asciidoc | 53 +++++++++++++----- .../setup-aks-provisioning-pipeline.asciidoc | 54 +++++++++++++------ 2 files changed, 77 insertions(+), 30 deletions(-) diff --git a/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc b/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc index 07d0dd68d..f3966a96b 100644 --- a/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc @@ -1,17 +1,25 @@ -= Setting up a Azure AKS provisioning pipeline on Azure DevOps +:provider: Azure Devops +:pipeline_type: pipeline +:trigger_sentence_azure: +:pipeline_type2: pipeline +:path_provider: azure-devops +:aks_variables_path: Azure DevOps > Pipelines > Library > 'aks-variables += Setting up a Azure AKS provisioning {pipeline_type} on {provider} -In this section we will create a pipeline which will provision an Azure AKS cluster. This pipeline will be configured to be manually triggered by the user. As part of AKS cluster provisioning, a NGINX Ingress controller is deployed and a variable group with the name `aks-variables` is created, which contains, among others, the DNS name of the Ingress controller, that you you will need to add as CNAME record on the domains used in your application Ingress manifest files. Refer to the appendix to retrieve the DNS name of the Ingress controller independently. +In this section we will create a {pipeline_type} which will provision an Azure AKS cluster. This {pipeline_type} will be configured to be manually triggered by the user. As part of AKS cluster provisioning, a NGINX Ingress controller is deployed and a variable group with the name `aks-variables` is created, which contains, among others, the DNS name of the Ingress controller, that you you will need to add as CNAME record on the domains used in your application Ingress manifest files. Refer to the appendix for more details. -The creation of the pipeline will follow the project workflow, so a new branch named `feature/aks-provisioning` will be created, the YAML file for the pipeline and the terraform files for creating the cluster will be pushed to it. +The creation of the {pipeline_type} will follow the project workflow, so a new branch named `feature/aks-provisioning` will be created, the YAML file for the {pipeline_type} and the terraform files for creating the cluster will be pushed to it. Then, a Pull Request (PR) will be created in order to merge the new branch into the appropiate branch (provided in `-b` flag). The PR will be automatically merged if the repository policies are met. If the merge is not possible, either the PR URL will be shown as output, or it will be opened in your web browser if using `-w` flag. -The script located at `/scripts/pipelines/azure-devops/pipeline_generator.sh` will automatically create this new branch, create the AKS provisioning pipeline based on the YAML template, create the Pull Request and, if it is possible, merge this new branch into the specified branch. +The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` will automatically create this new branch, create the AKS provisioning {pipeline_type} based on the YAML template, create the Pull Request and, if it is possible, merge this new branch into the specified branch. == Prerequisites -* Install the https://marketplace.visualstudio.com/items?itemName=ms-devlabs.custom-terraform-tasks[Terraform extension] for Azure DevOps. -* Create a https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#create-a-service-connection[service connection] to Azure Resource Manager and name it `aks-connection`. If you already have a service connection available or you need a specific connection name, please update `aks-pipeline.cfg` accordingly. + * Install the https://marketplace.visualstudio.com/items?itemName=ms-devlabs.custom-terraform-tasks[Terraform extension] for Azure DevOps. + * Create a https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#create-a-service-connection[service connection] to Azure Resource Manager and name it `aks-connection`. If you already have a service connection available or you need a specific connection name, please update `aks-pipeline.cfg` accordingly. + + * An Azure resource group in the desired cluster location (e.g. `westeurope`). You can use an existing one or create a new one with the following command: ``` @@ -32,12 +40,12 @@ az storage container create -n --account-name --location --resource_group_name --instance_type --worker_node_count --dns_prefix +./set-terraform-variables.sh --location --resource_group_name --instance_type --worker_node_count --dns_prefix ``` === Usage @@ -46,23 +54,27 @@ pipeline_generator.sh \ -c \ -n \ -d \ + --cluster-name \ --resource-group \ --storage-account \ --storage-container \ + [--rancher] \ [-b ] \ [-w] ``` -NOTE: The config file for the AKS provisioning pipeline is located at `/scripts/pipelines/azure-devops/templates/aks/aks-pipeline.cfg`. +NOTE: The config file for the AKS provisioning {pipeline_type} is located at `/scripts/pipelines/{path_provider}/templates/aks/aks-pipeline.cfg`. === Flags ``` -c, --config-file [Required] Configuration file containing pipeline definition. -n, --pipeline-name [Required] Name that will be set to the pipeline. -d, --local-directory [Required] Local directory of your project (the path should always be using '/' and not '\'). - --resource-group [Required] Name of the resource group for the cluster. - --storage--account [Required] Name of the storage account for the cluster. + --cluster-name [Required] Name for the cluster. + --resource-group [Required] Name of the resource group for the cluster. + --storage--account [Required] Name of the storage account for the cluster. --storage-container [Required] Name of the storage container where the Terraform state of the cluster will be stored. + --rancher Install Rancher to manage the cluster. -b, --target-branch Name of the branch to which the Pull Request will target. PR is not created if the flag is not provided. -w Open the Pull Request on the web browser if it cannot be automatically merged. Requires -b flag. ``` @@ -70,14 +82,17 @@ NOTE: The config file for the AKS provisioning pipeline is located at `/scripts/ === Example ``` -./pipeline_generator.sh -c ./templates/aks/aks-pipeline.cfg -n aks-provisioning -d C:/Users/$USERNAME/Desktop/quarkus-project --resource-group devonfw --storage-account hangar --storage-container aks-state -b develop -w + ./pipeline_generator.sh -c ./templates/aks/aks-pipeline.cfg -n aks-provisioning -d C:/Users/$USERNAME/Desktop/quarkus-project --cluster-name devon-hangar --resource-group devonfw --storage-account hangar --storage-container aks-state --rancher -b develop -w + ``` +NOTE: Rancher is installed on the cluster after provisioning when using the above command. + === Appendix: Interacting with the cluster NOTE: Make sure you have https://kubernetes.io/docs/tasks/tools/#kubectl[kubectl] installed. -In order to interact with your cluster you will need to download the artifact `kubeconfig` generated by the cluster provisioning pipeline on the location it is expected by default (`~/.kube/config`) or either: +In order to interact with your cluster you will need to download the artifact `kubeconfig` generated by the cluster provisioning {pipeline_type} on the location it is expected by default (`~/.kube/config`) or either: ``` # via environment variable (you can add this on your profile) @@ -87,3 +102,15 @@ kubectl # via command-line flag kubectl --kubeconfig= ``` + +To get the DNS name of the NGINX Ingress controller on the AKS cluster, go into {aks_variables_path}. + +Rancher, if installed, will be available on `https:///dashboard`. You will be asked for an initial password, which can be retrieved with: + +``` +kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}{{"\n"}}' +``` + +=== Appendix: Destroying the cluster + +To destroy the provisioned resources, set `operation` {pipeline_type} variable value to `destroy` and run the {pipeline_type}. diff --git a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc index 07d0dd68d..4d6c46648 100644 --- a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc @@ -1,17 +1,24 @@ -= Setting up a Azure AKS provisioning pipeline on Azure DevOps +:provider: Gitlab +:pipeline_type: pipeline +:trigger_sentence_gitlab: +:pipeline_type2: gitlab +:path_provider: gitlab +:aks_variables_path: Group > settings > ci_cd > Variables += Setting up a Azure AKS provisioning {pipeline_type} on {provider} -In this section we will create a pipeline which will provision an Azure AKS cluster. This pipeline will be configured to be manually triggered by the user. As part of AKS cluster provisioning, a NGINX Ingress controller is deployed and a variable group with the name `aks-variables` is created, which contains, among others, the DNS name of the Ingress controller, that you you will need to add as CNAME record on the domains used in your application Ingress manifest files. Refer to the appendix to retrieve the DNS name of the Ingress controller independently. +In this section we will create a {pipeline_type} which will provision an Azure AKS cluster. This {pipeline_type} will be configured to be manually triggered by the user. As part of AKS cluster provisioning, a NGINX Ingress controller is deployed and a variable group with the name `aks-variables` is created, which contains, among others, the DNS name of the Ingress controller, that you you will need to add as CNAME record on the domains used in your application Ingress manifest files. Refer to the appendix for more details. -The creation of the pipeline will follow the project workflow, so a new branch named `feature/aks-provisioning` will be created, the YAML file for the pipeline and the terraform files for creating the cluster will be pushed to it. +The creation of the {pipeline_type} will follow the project workflow, so a new branch named `feature/aks-provisioning` will be created, the YAML file for the {pipeline_type} and the terraform files for creating the cluster will be pushed to it. Then, a Pull Request (PR) will be created in order to merge the new branch into the appropiate branch (provided in `-b` flag). The PR will be automatically merged if the repository policies are met. If the merge is not possible, either the PR URL will be shown as output, or it will be opened in your web browser if using `-w` flag. -The script located at `/scripts/pipelines/azure-devops/pipeline_generator.sh` will automatically create this new branch, create the AKS provisioning pipeline based on the YAML template, create the Pull Request and, if it is possible, merge this new branch into the specified branch. +The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` will automatically create this new branch, create the AKS provisioning {pipeline_type} based on the YAML template, create the Pull Request and, if it is possible, merge this new branch into the specified branch. == Prerequisites -* Install the https://marketplace.visualstudio.com/items?itemName=ms-devlabs.custom-terraform-tasks[Terraform extension] for Azure DevOps. -* Create a https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#create-a-service-connection[service connection] to Azure Resource Manager and name it `aks-connection`. If you already have a service connection available or you need a specific connection name, please update `aks-pipeline.cfg` accordingly. + + * Add AZURE credentials as [Variables] in your repository and name it `AZURE_USERNAME`, `AZURE_PASSWORD`. If you already have a available credentials or you need a specific credentials connection, please update `aks-provisioning.yml` accordingly. + * An Azure resource group in the desired cluster location (e.g. `westeurope`). You can use an existing one or create a new one with the following command: ``` @@ -32,37 +39,35 @@ az storage container create -n --account-name --location --resource_group_name --instance_type --worker_node_count --dns_prefix +./set-terraform-variables.sh --location --resource_group_name --instance_type --worker_node_count --dns_prefix ``` === Usage ``` pipeline_generator.sh \ -c \ - -n \ -d \ - --resource-group \ - --storage-account \ + --cluster-name \ --storage-container \ + [--rancher] \ [-b ] \ [-w] ``` -NOTE: The config file for the AKS provisioning pipeline is located at `/scripts/pipelines/azure-devops/templates/aks/aks-pipeline.cfg`. +NOTE: The config file for the AKS provisioning {pipeline_type} is located at `/scripts/pipelines/{path_provider}/templates/aks/aks-pipeline.cfg`. === Flags ``` -c, --config-file [Required] Configuration file containing pipeline definition. --n, --pipeline-name [Required] Name that will be set to the pipeline. -d, --local-directory [Required] Local directory of your project (the path should always be using '/' and not '\'). - --resource-group [Required] Name of the resource group for the cluster. - --storage--account [Required] Name of the storage account for the cluster. + --cluster-name [Required] Name for the cluster. --storage-container [Required] Name of the storage container where the Terraform state of the cluster will be stored. + --rancher Install Rancher to manage the cluster. -b, --target-branch Name of the branch to which the Pull Request will target. PR is not created if the flag is not provided. -w Open the Pull Request on the web browser if it cannot be automatically merged. Requires -b flag. ``` @@ -70,14 +75,17 @@ NOTE: The config file for the AKS provisioning pipeline is located at `/scripts/ === Example ``` -./pipeline_generator.sh -c ./templates/aks/aks-pipeline.cfg -n aks-provisioning -d C:/Users/$USERNAME/Desktop/quarkus-project --resource-group devonfw --storage-account hangar --storage-container aks-state -b develop -w + + ./pipeline_generator.sh -c ./templates/aks/aks-pipeline.cfg -n aks-provisioning -d C:/Users/$USERNAME/Desktop/quarkus-project --cluster-name devon-hangar --storage-container aks-state --rancher -b develop -w ``` +NOTE: Rancher is installed on the cluster after provisioning when using the above command. + === Appendix: Interacting with the cluster NOTE: Make sure you have https://kubernetes.io/docs/tasks/tools/#kubectl[kubectl] installed. -In order to interact with your cluster you will need to download the artifact `kubeconfig` generated by the cluster provisioning pipeline on the location it is expected by default (`~/.kube/config`) or either: +In order to interact with your cluster you will need to download the artifact `kubeconfig` generated by the cluster provisioning {pipeline_type} on the location it is expected by default (`~/.kube/config`) or either: ``` # via environment variable (you can add this on your profile) @@ -87,3 +95,15 @@ kubectl # via command-line flag kubectl --kubeconfig= ``` + +To get the DNS name of the NGINX Ingress controller on the AKS cluster, go into {aks_variables_path}. + +Rancher, if installed, will be available on `https:///dashboard`. You will be asked for an initial password, which can be retrieved with: + +``` +kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}{{"\n"}}' +``` + +=== Appendix: Destroying the cluster + +To destroy the provisioned resources, set `operation` {pipeline_type} variable value to `destroy` and run the {pipeline_type}. From d59666dcb5d0658aeaa6217d14c1471629d19dd1 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 12 Sep 2022 12:08:50 +0530 Subject: [PATCH 14/35] Update setup-aks-provisioning-pipeline.asciidoc --- .../setup-aks-provisioning-pipeline.asciidoc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc b/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc index c40247101..54b507ce9 100644 --- a/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc @@ -20,7 +20,7 @@ ifdef::trigger_sentence_gitlab[ * Add AZURE credentials as [Variables] in your r ``` az group create -n -l ``` - +ifndef::trigger_sentence_azure,trigger_sentence_github[] * An Azure storage account within the previous resource group. You can use an existing one or create a new one with the following command: ``` @@ -32,6 +32,7 @@ az storage account create -n -g -l ``` az storage container create -n --account-name ``` +endif::[] * This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled the latest changes with `git pull`). @@ -109,4 +110,4 @@ kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{ === Appendix: Destroying the cluster -To destroy the provisioned resources, set `operation` {pipeline_type} variable value to `destroy` and run the {pipeline_type}. \ No newline at end of file +To destroy the provisioned resources, set `operation` {pipeline_type} variable value to `destroy` and run the {pipeline_type}. From 319380c60df9918daff40e92b63acfb161ed1abb Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 12 Sep 2022 06:39:08 +0000 Subject: [PATCH 15/35] automatic commit dor docs --- .../setup-aks-provisioning-pipeline.asciidoc | 12 ------------ .../gitlab/setup-aks-provisioning-pipeline.asciidoc | 1 - 2 files changed, 13 deletions(-) diff --git a/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc b/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc index f3966a96b..fb8511316 100644 --- a/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc @@ -26,18 +26,6 @@ The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` az group create -n -l ``` -* An Azure storage account within the previous resource group. You can use an existing one or create a new one with the following command: - -``` -az storage account create -n -g -l -``` - -* An Azure storage container in Azure within the previous storage account. You can use an existing one or create a new one with the following command: - -``` -az storage container create -n --account-name -``` - * This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled the latest changes with `git pull`). == Creating the {pipeline_type} using provided script diff --git a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc index 4d6c46648..91da0956b 100644 --- a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc @@ -24,7 +24,6 @@ The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` ``` az group create -n -l ``` - * An Azure storage account within the previous resource group. You can use an existing one or create a new one with the following command: ``` From cf5e4faf6633b45945a1ad1824f48daae896ecb4 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 12 Sep 2022 12:17:40 +0530 Subject: [PATCH 16/35] Update setup-aks-provisioning-pipeline.asciidoc --- .../common_templates/setup-aks-provisioning-pipeline.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc b/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc index 54b507ce9..3b4123869 100644 --- a/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc @@ -21,6 +21,7 @@ ifdef::trigger_sentence_gitlab[ * Add AZURE credentials as [Variables] in your r az group create -n -l ``` ifndef::trigger_sentence_azure,trigger_sentence_github[] + * An Azure storage account within the previous resource group. You can use an existing one or create a new one with the following command: ``` @@ -80,7 +81,7 @@ ifdef::trigger_sentence_azure,trigger_sentence_github[ --storage--account [ ``` ifdef::trigger_sentence_azure,trigger_sentence_github[ ./pipeline_generator.sh -c ./templates/aks/aks-pipeline.cfg -n aks-provisioning -d C:/Users/$USERNAME/Desktop/quarkus-project --cluster-name devon-hangar --resource-group devonfw --storage-account hangar --storage-container aks-state --rancher -b develop -w ] -ifdef::trigger_sentence_gitlab[ ./pipeline_generator.sh -c ./templates/aks/aks-pipeline.cfg -n aks-provisioning -d C:/Users/$USERNAME/Desktop/quarkus-project --cluster-name devon-hangar --storage-container aks-state --rancher -b develop -w ] +ifdef::trigger_sentence_gitlab[ ./pipeline_generator.sh -c ./templates/aks/aks-pipeline.cfg -d C:/Users/$USERNAME/Desktop/quarkus-project --cluster-name devon-hangar --storage-container aks-state --rancher -b develop -w ] ``` NOTE: Rancher is installed on the cluster after provisioning when using the above command. From 601df901307300958315d5ef8a1bfceba57700dc Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 12 Sep 2022 06:48:01 +0000 Subject: [PATCH 17/35] automatic commit dor docs --- documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc index 91da0956b..57e0fbfe9 100644 --- a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc @@ -24,6 +24,7 @@ The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` ``` az group create -n -l ``` + * An Azure storage account within the previous resource group. You can use an existing one or create a new one with the following command: ``` @@ -75,7 +76,7 @@ NOTE: The config file for the AKS provisioning {pipeline_type} is located at `/s ``` - ./pipeline_generator.sh -c ./templates/aks/aks-pipeline.cfg -n aks-provisioning -d C:/Users/$USERNAME/Desktop/quarkus-project --cluster-name devon-hangar --storage-container aks-state --rancher -b develop -w + ./pipeline_generator.sh -c ./templates/aks/aks-pipeline.cfg -d C:/Users/$USERNAME/Desktop/quarkus-project --cluster-name devon-hangar --storage-container aks-state --rancher -b develop -w ``` NOTE: Rancher is installed on the cluster after provisioning when using the above command. From eac3b5df414a190de771875833f3b9d95f7fec2d Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 12 Sep 2022 14:02:39 +0530 Subject: [PATCH 18/35] Update setup-aks-provisioning-pipeline.asciidoc --- .../common_templates/setup-aks-provisioning-pipeline.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc b/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc index 3b4123869..7ce71b301 100644 --- a/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc @@ -13,14 +13,14 @@ The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` ifdef::trigger_sentence_azure[ * Install the https://marketplace.visualstudio.com/items?itemName=ms-devlabs.custom-terraform-tasks[Terraform extension] for Azure DevOps.] ifdef::trigger_sentence_azure[ * Create a https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#create-a-service-connection[service connection] to Azure Resource Manager and name it `aks-connection`. If you already have a service connection available or you need a specific connection name, please update `aks-pipeline.cfg` accordingly.] -ifdef::trigger_sentence_gitlab[ * Add AZURE credentials as [Variables] in your repository and name it `AZURE_USERNAME`, `AZURE_PASSWORD`. If you already have a available credentials or you need a specific credentials connection, please update `aks-provisioning.yml` accordingly.] +ifdef::trigger_sentence_gitlab[ * Add AZURE credentials as https://docs.gitlab.com/ee/ci/variables/#add-a-cicd-variable-to-a-project[Variables] in your repository and name it `AZURE_USERNAME`, `AZURE_PASSWORD`. If you already have a available credentials or you need a specific credentials connection, please update `aks-provisioning.yml` accordingly.] * An Azure resource group in the desired cluster location (e.g. `westeurope`). You can use an existing one or create a new one with the following command: ``` az group create -n -l ``` -ifndef::trigger_sentence_azure,trigger_sentence_github[] +ifndef::trigger_sentence_gitlab[] * An Azure storage account within the previous resource group. You can use an existing one or create a new one with the following command: From 6f05bd107a0b7f5e36789e1d5b701dc396238460 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 12 Sep 2022 08:32:56 +0000 Subject: [PATCH 19/35] automatic commit dor docs --- .../setup-aks-provisioning-pipeline.asciidoc | 12 ++++++++++++ .../setup-aks-provisioning-pipeline.asciidoc | 14 +------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc b/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc index fb8511316..f3966a96b 100644 --- a/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc @@ -26,6 +26,18 @@ The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` az group create -n -l ``` +* An Azure storage account within the previous resource group. You can use an existing one or create a new one with the following command: + +``` +az storage account create -n -g -l +``` + +* An Azure storage container in Azure within the previous storage account. You can use an existing one or create a new one with the following command: + +``` +az storage container create -n --account-name +``` + * This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled the latest changes with `git pull`). == Creating the {pipeline_type} using provided script diff --git a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc index 57e0fbfe9..0b6a30d02 100644 --- a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc @@ -17,7 +17,7 @@ The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` == Prerequisites - * Add AZURE credentials as [Variables] in your repository and name it `AZURE_USERNAME`, `AZURE_PASSWORD`. If you already have a available credentials or you need a specific credentials connection, please update `aks-provisioning.yml` accordingly. + * Add AZURE credentials as https://docs.gitlab.com/ee/ci/variables/#add-a-cicd-variable-to-a-project[Variables] in your repository and name it `AZURE_USERNAME`, `AZURE_PASSWORD`. If you already have a available credentials or you need a specific credentials connection, please update `aks-provisioning.yml` accordingly. * An Azure resource group in the desired cluster location (e.g. `westeurope`). You can use an existing one or create a new one with the following command: @@ -25,18 +25,6 @@ The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` az group create -n -l ``` -* An Azure storage account within the previous resource group. You can use an existing one or create a new one with the following command: - -``` -az storage account create -n -g -l -``` - -* An Azure storage container in Azure within the previous storage account. You can use an existing one or create a new one with the following command: - -``` -az storage container create -n --account-name -``` - * This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled the latest changes with `git pull`). == Creating the {pipeline_type} using provided script From 85a1d85ff3307d1c71eec09deba90fdb49b002d6 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 12 Sep 2022 14:04:16 +0530 Subject: [PATCH 20/35] Update setup-aks-provisioning-pipeline.asciidoc --- .../common_templates/setup-aks-provisioning-pipeline.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc b/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc index 7ce71b301..1e36a3c1c 100644 --- a/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc @@ -20,7 +20,7 @@ ifdef::trigger_sentence_gitlab[ * Add AZURE credentials as https://docs.gitlab.c ``` az group create -n -l ``` -ifndef::trigger_sentence_gitlab[] +ifndef::trigger_sentence_gitlab[ * An Azure storage account within the previous resource group. You can use an existing one or create a new one with the following command: @@ -33,7 +33,7 @@ az storage account create -n -g -l ``` az storage container create -n --account-name ``` -endif::[] +] * This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled the latest changes with `git pull`). From 44bd0265c6c7c1af23515411a2ae134b63cab4dd Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 12 Sep 2022 08:34:33 +0000 Subject: [PATCH 21/35] automatic commit dor docs --- .../setup-aks-provisioning-pipeline.asciidoc | 2 ++ .../setup-aks-provisioning-pipeline.asciidoc | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc b/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc index f3966a96b..3b001371a 100644 --- a/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc @@ -25,6 +25,7 @@ The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` ``` az group create -n -l ``` +ifndef::trigger_sentence_gitlab[ * An Azure storage account within the previous resource group. You can use an existing one or create a new one with the following command: @@ -37,6 +38,7 @@ az storage account create -n -g -l ``` az storage container create -n --account-name ``` +] * This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled the latest changes with `git pull`). diff --git a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc index 0b6a30d02..aaa03bb9e 100644 --- a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc @@ -24,6 +24,20 @@ The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` ``` az group create -n -l ``` +ifndef::trigger_sentence_gitlab[ + +* An Azure storage account within the previous resource group. You can use an existing one or create a new one with the following command: + +``` +az storage account create -n -g -l +``` + +* An Azure storage container in Azure within the previous storage account. You can use an existing one or create a new one with the following command: + +``` +az storage container create -n --account-name +``` +] * This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled the latest changes with `git pull`). From 42b59d036509df144f1a17258852c758d23d4716 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 12 Sep 2022 14:06:30 +0530 Subject: [PATCH 22/35] Update setup-aks-provisioning-pipeline.asciidoc --- .../common_templates/setup-aks-provisioning-pipeline.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc b/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc index 1e36a3c1c..7ce71b301 100644 --- a/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc @@ -20,7 +20,7 @@ ifdef::trigger_sentence_gitlab[ * Add AZURE credentials as https://docs.gitlab.c ``` az group create -n -l ``` -ifndef::trigger_sentence_gitlab[ +ifndef::trigger_sentence_gitlab[] * An Azure storage account within the previous resource group. You can use an existing one or create a new one with the following command: @@ -33,7 +33,7 @@ az storage account create -n -g -l ``` az storage container create -n --account-name ``` -] +endif::[] * This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled the latest changes with `git pull`). From 3183a7bee22f7ae2c64888267c0038580e287336 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 12 Sep 2022 08:36:49 +0000 Subject: [PATCH 23/35] automatic commit dor docs --- .../setup-aks-provisioning-pipeline.asciidoc | 2 -- .../setup-aks-provisioning-pipeline.asciidoc | 14 -------------- 2 files changed, 16 deletions(-) diff --git a/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc b/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc index 3b001371a..f3966a96b 100644 --- a/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc @@ -25,7 +25,6 @@ The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` ``` az group create -n -l ``` -ifndef::trigger_sentence_gitlab[ * An Azure storage account within the previous resource group. You can use an existing one or create a new one with the following command: @@ -38,7 +37,6 @@ az storage account create -n -g -l ``` az storage container create -n --account-name ``` -] * This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled the latest changes with `git pull`). diff --git a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc index aaa03bb9e..0b6a30d02 100644 --- a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc @@ -24,20 +24,6 @@ The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` ``` az group create -n -l ``` -ifndef::trigger_sentence_gitlab[ - -* An Azure storage account within the previous resource group. You can use an existing one or create a new one with the following command: - -``` -az storage account create -n -g -l -``` - -* An Azure storage container in Azure within the previous storage account. You can use an existing one or create a new one with the following command: - -``` -az storage container create -n --account-name -``` -] * This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled the latest changes with `git pull`). From a196af994b55fadd598617cc5bca2ddcb0719332 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 12 Sep 2022 14:18:46 +0530 Subject: [PATCH 24/35] Update aks-provisioning.yml.template --- .../gitlab/templates/aks/aks-provisioning.yml.template | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template index 46ad50f8d..37193fd76 100644 --- a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template +++ b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template @@ -9,7 +9,7 @@ workflow: rules: - if: '$CI_PIPELINE_SOURCE == "web"' when: always - - when: never + - when: always variables: CLUSTER_NAME: @@ -83,6 +83,9 @@ Provision: - mv ${TF_ROOT}/kubeconfig ${CI_PROJECT_DIR} - *glab_auth - cat ${CI_PROJECT_DIR}/kubeconfig | glab variable set -g devon-hangar KUBECONFIG -t "file" + artifacts: + paths: + - "./kubeconfig" rules: - if: '$OPERATION == "create"' when: always From 84bd8705be74ffb856f924795f9349e9a36db1d9 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 12 Sep 2022 09:05:57 +0000 Subject: [PATCH 25/35] Automatic generation of documentation --- .../azure-devops/setup-aks-provisioning-pipeline.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc b/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc index fd8345c67..f3966a96b 100644 --- a/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc @@ -83,6 +83,7 @@ NOTE: The config file for the AKS provisioning {pipeline_type} is located at `/s ``` ./pipeline_generator.sh -c ./templates/aks/aks-pipeline.cfg -n aks-provisioning -d C:/Users/$USERNAME/Desktop/quarkus-project --cluster-name devon-hangar --resource-group devonfw --storage-account hangar --storage-container aks-state --rancher -b develop -w + ``` NOTE: Rancher is installed on the cluster after provisioning when using the above command. From f300186d072e2b259b4a8db39aa9079f068f9378 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Tue, 13 Sep 2022 13:54:27 +0530 Subject: [PATCH 26/35] updated with minor changes --- .../setup-aks-provisioning-pipeline.asciidoc | 2 ++ scripts/pipelines/gitlab/pipeline_generator.sh | 4 +--- scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg | 2 +- scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml | 6 ++++++ 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc b/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc index 7ce71b301..5e0b0d86b 100644 --- a/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/src/common_templates/setup-aks-provisioning-pipeline.asciidoc @@ -15,6 +15,8 @@ ifdef::trigger_sentence_azure[ * Create a https://docs.microsoft.com/en-us/azure ifdef::trigger_sentence_gitlab[ * Add AZURE credentials as https://docs.gitlab.com/ee/ci/variables/#add-a-cicd-variable-to-a-project[Variables] in your repository and name it `AZURE_USERNAME`, `AZURE_PASSWORD`. If you already have a available credentials or you need a specific credentials connection, please update `aks-provisioning.yml` accordingly.] +ifdef::trigger_sentence_gitlab[* Create a Gitlab personal access token https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token[PAT], Store in environment variable as `GITLAB_PAT`.] + * An Azure resource group in the desired cluster location (e.g. `westeurope`). You can use an existing one or create a new one with the following command: ``` diff --git a/scripts/pipelines/gitlab/pipeline_generator.sh b/scripts/pipelines/gitlab/pipeline_generator.sh index 2ca9bcb0d..e306f8dd6 100644 --- a/scripts/pipelines/gitlab/pipeline_generator.sh +++ b/scripts/pipelines/gitlab/pipeline_generator.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e -FLAGS=$(getopt -a --options c:n:d:a:b:l:i:u:p:hw --long "config-file:,pipeline-name:,local-directory:,artifact-path:,target-branch:,language:,build-pipeline-name:,sonar-url:,sonar-token:,image-name:,registry-user:,registry-password:,resource-group:,storage-account:,storage-container:,cluster-name:,s3-bucket:,s3-key-path:,quality-pipeline-name:,dockerfile:,test-pipeline-name:,aws-access-key:,aws-secret-access-key:,aws-region:,help,rancher" -- "$@") +FLAGS=$(getopt -a --options c:n:d:a:b:l:i:u:p:hw --long "config-file:,pipeline-name:,local-directory:,artifact-path:,target-branch:,language:,build-pipeline-name:,sonar-url:,sonar-token:,image-name:,registry-user:,registry-password:,storage-container:,cluster-name:,s3-bucket:,s3-key-path:,quality-pipeline-name:,dockerfile:,test-pipeline-name:,aws-access-key:,aws-secret-access-key:,aws-region:,help,rancher" -- "$@") eval set -- "$FLAGS" while true; do @@ -17,8 +17,6 @@ while true; do -i | --image-name) imageName=$2; shift 2;; -u | --registry-user) dockerUser=$2; shift 2;; -p | --registry-password) dockerPassword=$2; shift 2;; - --resource-group) resourceGroupName=$2; shift 2;; - --storage-account) storageAccountName=$2; shift 2;; --storage-container) storageContainerName=$2; shift 2;; --rancher) installRancher="true"; shift 1;; --cluster-name) clusterName=$2; shift 2;; diff --git a/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg b/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg index 1f58834d4..bdc622112 100644 --- a/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg +++ b/scripts/pipelines/gitlab/templates/aks/aks-pipeline.cfg @@ -54,5 +54,5 @@ function addPipelineVariables { export storageContainerName export installRancher export operation - specificEnvSubstList='${clusterName} ${operation} ${storageContainerName} ${installRancher}' #${resourceGroupName} ${storageAccountName} + specificEnvSubstList='${clusterName} ${operation} ${storageContainerName} ${installRancher}' } diff --git a/scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml b/scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml index 06d54fb5b..c38d0535e 100644 --- a/scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml +++ b/scripts/pipelines/gitlab/templates/common/.gitlab-ci.yml @@ -1,6 +1,12 @@ include: - '.pipelines/*.yml' +workflow: + rules: + - if: '$CI_PIPELINE_SOURCE == "web"' + when: always + - when: never + # stages: # - build # - test From 09f064e87fb37e2496e315d1567a3a98ec1bff22 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Tue, 13 Sep 2022 08:25:00 +0000 Subject: [PATCH 27/35] Automatic generation of documentation --- .../azure-devops/setup-aks-provisioning-pipeline.asciidoc | 1 + documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc | 2 ++ 2 files changed, 3 insertions(+) diff --git a/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc b/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc index f3966a96b..5b712596b 100644 --- a/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/azure-devops/setup-aks-provisioning-pipeline.asciidoc @@ -20,6 +20,7 @@ The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` * Create a https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#create-a-service-connection[service connection] to Azure Resource Manager and name it `aks-connection`. If you already have a service connection available or you need a specific connection name, please update `aks-pipeline.cfg` accordingly. + * An Azure resource group in the desired cluster location (e.g. `westeurope`). You can use an existing one or create a new one with the following command: ``` diff --git a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc index 0b6a30d02..af597083c 100644 --- a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc @@ -19,6 +19,8 @@ The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` * Add AZURE credentials as https://docs.gitlab.com/ee/ci/variables/#add-a-cicd-variable-to-a-project[Variables] in your repository and name it `AZURE_USERNAME`, `AZURE_PASSWORD`. If you already have a available credentials or you need a specific credentials connection, please update `aks-provisioning.yml` accordingly. +* Create a Gitlab personal access token https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token[PAT], Store in environment variable as `GITLAB_PAT`. + * An Azure resource group in the desired cluster location (e.g. `westeurope`). You can use an existing one or create a new one with the following command: ``` From e0bb0568874340c2fbd16f7cad4b886bae749f05 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Tue, 13 Sep 2022 13:57:19 +0530 Subject: [PATCH 28/35] Update setup-aks-provisioning-pipeline.asciidoc --- documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc index af597083c..bae4cb21d 100644 --- a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc @@ -19,7 +19,7 @@ The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` * Add AZURE credentials as https://docs.gitlab.com/ee/ci/variables/#add-a-cicd-variable-to-a-project[Variables] in your repository and name it `AZURE_USERNAME`, `AZURE_PASSWORD`. If you already have a available credentials or you need a specific credentials connection, please update `aks-provisioning.yml` accordingly. -* Create a Gitlab personal access token https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token[PAT], Store in environment variable as `GITLAB_PAT`. +* Create a Gitlab personal access token https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token[PAT], Store it as `GITLAB_PAT` environment variable. * An Azure resource group in the desired cluster location (e.g. `westeurope`). You can use an existing one or create a new one with the following command: From 5e01a15a86bd067d2d5981911ed75b0cae5e3219 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Tue, 13 Sep 2022 14:47:28 +0530 Subject: [PATCH 29/35] Update aks-provisioning.yml.template --- .../gitlab/templates/aks/aks-provisioning.yml.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template index 37193fd76..c8986028f 100644 --- a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template +++ b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template @@ -48,7 +48,7 @@ variables: # INSTALL AZCLI - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - az --version - - az login #--username "${AZURE_USERNAME}" --password "${AZURE_PASSWORD}" + - az login --username "${AZURE_USERNAME}" --password "${AZURE_PASSWORD}" .packages: &configure_packages # INSTALL KUBECTL From 93191dec5d7348818d51c9f8191602dae3ca567f Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Tue, 20 Sep 2022 10:38:09 +0530 Subject: [PATCH 30/35] Update aks-provisioning.yml.template --- .../gitlab/templates/aks/aks-provisioning.yml.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template index c8986028f..ac364c4f4 100644 --- a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template +++ b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template @@ -23,7 +23,7 @@ variables: description: "Installs Rancher on AKS when set to true." TF_STATE_NAME: "$storageContainerName" TF_CACHE_KEY: default - TF_ROOT: "${CI_PROJECT_DIR}/azure/aks" + TF_ROOT: "${CI_PROJECT_DIR}/.terraform/aks" TF_USERNAME: ${GITLAB_USER_NAME} TF_PASSWORD: ${GITLAB_PAT} TF_ADDRESS: "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/terraform/state/${TF_STATE_NAME}" From 67128fe16bba89d0f5c4a027db5f5e7c2d276964 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Wed, 21 Sep 2022 10:19:23 +0530 Subject: [PATCH 31/35] Update aks-provisioning.yml.template --- .../gitlab/templates/aks/aks-provisioning.yml.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template index ac364c4f4..27b11dd3b 100644 --- a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template +++ b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template @@ -14,7 +14,7 @@ workflow: variables: CLUSTER_NAME: value: "$clusterName" - description: "Name for the AWS EKS cluster to be created" + description: "Name for the AKS cluster to be created" OPERATION: value: "$operation" description: "Operation to perform on cluster. Create or Destroy." From 54b952cb767569777a3f2b0900be85079f5a3f07 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Thu, 22 Sep 2022 10:54:45 +0530 Subject: [PATCH 32/35] Update setup-aks-provisioning-pipeline.asciidoc --- .../src/gitlab/setup-aks-provisioning-pipeline.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/src/gitlab/setup-aks-provisioning-pipeline.asciidoc b/documentation/src/gitlab/setup-aks-provisioning-pipeline.asciidoc index 468c977ec..bfa0e86fd 100644 --- a/documentation/src/gitlab/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/src/gitlab/setup-aks-provisioning-pipeline.asciidoc @@ -1,7 +1,7 @@ :provider: Gitlab :pipeline_type: pipeline :trigger_sentence_gitlab: -:pipeline_type2: gitlab +:pipeline_type2: Gitlab pipeline :path_provider: gitlab :aks_variables_path: Group > settings > ci_cd > Variables -include::../common_templates/setup-aks-provisioning-pipeline.asciidoc[] \ No newline at end of file +include::../common_templates/setup-aks-provisioning-pipeline.asciidoc[] From 416489216bf7851b0e3179113fff7ab3fa4f7460 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Thu, 22 Sep 2022 05:25:10 +0000 Subject: [PATCH 33/35] Automatic generation of documentation --- documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc index bae4cb21d..5fd710279 100644 --- a/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc +++ b/documentation/gitlab/setup-aks-provisioning-pipeline.asciidoc @@ -1,7 +1,7 @@ :provider: Gitlab :pipeline_type: pipeline :trigger_sentence_gitlab: -:pipeline_type2: gitlab +:pipeline_type2: Gitlab pipeline :path_provider: gitlab :aks_variables_path: Group > settings > ci_cd > Variables = Setting up a Azure AKS provisioning {pipeline_type} on {provider} @@ -19,7 +19,7 @@ The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` * Add AZURE credentials as https://docs.gitlab.com/ee/ci/variables/#add-a-cicd-variable-to-a-project[Variables] in your repository and name it `AZURE_USERNAME`, `AZURE_PASSWORD`. If you already have a available credentials or you need a specific credentials connection, please update `aks-provisioning.yml` accordingly. -* Create a Gitlab personal access token https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token[PAT], Store it as `GITLAB_PAT` environment variable. +* Create a Gitlab personal access token https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token[PAT], Store in environment variable as `GITLAB_PAT`. * An Azure resource group in the desired cluster location (e.g. `westeurope`). You can use an existing one or create a new one with the following command: From 000ceaf205ce4fb5cb33d1281e813f1f1b68baf2 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Thu, 22 Sep 2022 15:25:10 +0530 Subject: [PATCH 34/35] updated --- .../gitlab/templates/aks/aks-provisioning.yml.template | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template index 27b11dd3b..f2656298e 100644 --- a/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template +++ b/scripts/pipelines/gitlab/templates/aks/aks-provisioning.yml.template @@ -20,7 +20,8 @@ variables: description: "Operation to perform on cluster. Create or Destroy." INSTALL_RANCHER: value: "$installRancher" - description: "Installs Rancher on AKS when set to true." + description: "Installs Rancher on AKS when set to true." + organization: "$CI_PROJECT_NAMESPACE" TF_STATE_NAME: "$storageContainerName" TF_CACHE_KEY: default TF_ROOT: "${CI_PROJECT_DIR}/.terraform/aks" @@ -82,7 +83,7 @@ Provision: - terraform apply -var cluster_name=${CLUSTER_NAME} --auto-approve - mv ${TF_ROOT}/kubeconfig ${CI_PROJECT_DIR} - *glab_auth - - cat ${CI_PROJECT_DIR}/kubeconfig | glab variable set -g devon-hangar KUBECONFIG -t "file" + - cat ${CI_PROJECT_DIR}/kubeconfig | glab variable set -g $organization KUBECONFIG -t "file" artifacts: paths: - "./kubeconfig" @@ -115,7 +116,7 @@ Obtain_dns: - dnsname="${CLUSTER_NAME}.westeurope.cloudapp.azure.com" # Create aks_dns_name variable - *glab_auth - - glab variable set -g devon-hangar aks_dns_name -v "$dnsname" -t "env_var" + - glab variable set -g $organization aks_dns_name -v "$dnsname" -t "env_var" rules: - if: '$OPERATION == "create"' when: always From e322cf735ae4fde403697d82e4fcf8bd349068d5 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Tue, 18 Oct 2022 10:40:42 +0530 Subject: [PATCH 35/35] fixed with line ending checks --- .../common/kubernetes/install-nginx-ingress.sh | 6 +++--- .../templates/common/kubernetes/install-rancher.sh | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/pipelines/gitlab/templates/common/kubernetes/install-nginx-ingress.sh b/scripts/pipelines/gitlab/templates/common/kubernetes/install-nginx-ingress.sh index f735dd93b..0914cf122 100644 --- a/scripts/pipelines/gitlab/templates/common/kubernetes/install-nginx-ingress.sh +++ b/scripts/pipelines/gitlab/templates/common/kubernetes/install-nginx-ingress.sh @@ -1,4 +1,4 @@ -#!/bin/bash -helm repo add bitnami https://charts.bitnami.com/bitnami -helm repo update +#!/bin/bash +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo update helm install nginx-ingress bitnami/nginx-ingress-controller --set ingressClassResource.default=true --set containerSecurityContext.allowPrivilegeEscalation=false --namespace nginx-ingress --create-namespace \ No newline at end of file diff --git a/scripts/pipelines/gitlab/templates/common/kubernetes/install-rancher.sh b/scripts/pipelines/gitlab/templates/common/kubernetes/install-rancher.sh index eebe87ea9..66c61ac78 100644 --- a/scripts/pipelines/gitlab/templates/common/kubernetes/install-rancher.sh +++ b/scripts/pipelines/gitlab/templates/common/kubernetes/install-rancher.sh @@ -1,7 +1,7 @@ -#!/bin/bash -helm repo add rancher-latest https://releases.rancher.com/server-charts/latest -helm repo add jetstack https://charts.jetstack.io -helm repo add -kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.1/cert-manager.crds.yaml -helm install cert-manager "jetstack/cert-manager" --namespace cert-manager --create-namespace --version v1.5.1 +#!/bin/bash +helm repo add rancher-latest https://releases.rancher.com/server-charts/latest +helm repo add jetstack https://charts.jetstack.io +helm repo add +kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.1/cert-manager.crds.yaml +helm install cert-manager "jetstack/cert-manager" --namespace cert-manager --create-namespace --version v1.5.1 helm install rancher "rancher-latest/rancher" --namespace cattle-system --create-namespace --set hostname="$1" \ No newline at end of file