diff --git a/.gitignore b/.gitignore index 3a8fc31..94c699e 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ bin/ .idea/ *.iml out/ +build/ +output/ _site .sass-cache @@ -26,3 +28,13 @@ _site # Specific command files customers/* *.swp + +# 2git log files +2git-[-_0-9]*.log +logs +cc2git + +# ccmm2git files +projects.txt +*.env +project_baselines.txt diff --git a/_ccm-functions-test.sh b/_ccm-functions-test.sh new file mode 100644 index 0000000..47db9e8 --- /dev/null +++ b/_ccm-functions-test.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Load functions +source ./_ccm-functions.sh || source ${BASH_SOURCE%/*}/_ccm-functions.sh + +[[ "${debug:-}" == "true" ]] && set -x + + +set -euo pipefail + +ccm_project_name=" " +expected_result="-" +result="" +printf "%-8s: %-80s : %-75s " "test" "byref_translate_from_ccm_name2git_repo" "${ccm_project_name} -> $expected_result" +byref_translate_from_ccm_name2git_repo "$ccm_project_name" result +[[ "$result" == "${expected_result}" ]] && { printf "%10s\n" "SUCCESS" ; }|| { printf " FAILED: ${expected_result} != $result\n" ;} + +ccm_project_name="-" +expected_result="?" +result="" +printf "%-8s: %-80s : %-75s " "test" "byref_translate_from_git_repo2ccm_name_query" "${ccm_project_name} -> $expected_result" +byref_translate_from_git_repo2ccm_name_query $ccm_project_name result +[[ "$result" == "${expected_result}" ]] && { printf "%10s\n" "SUCCESS" ; }|| { printf " FAILED: ${expected_result} != $result\n" ;} + +ccm_query_name_instance_string="?" +ccm_query_instance="1" +expected_result=" " +result="" +printf "%-8s: %-80s : %-75s " "test" "byref_translate_from_ccm_name_instance_query2ccm_name" "${ccm_query_name_instance_string} -> $expected_result" +byref_translate_from_ccm_name_instance_query2ccm_name $ccm_query_name_instance_string $ccm_query_instance result +[[ "$result" == "${expected_result}" ]] && { printf "%10s\n" "SUCCESS" ; }|| { printf " FAILED: ${expected_result} != $result\n" ;} + +git_repo_name="-" +ccm_query_instance="1" +expected_result=" " +result="" +printf "%-8s: %-80s : %-75s " "test" "byref_translate_from_git_repo2ccm_name" "${ccm_project_name} -> $expected_result" +byref_translate_from_git_repo2ccm_name $git_repo_name $ccm_query_instance result +[[ "$result" == "${expected_result}" ]] && { printf "%10s\n" "SUCCESS" ; }|| { printf " FAILED: ${expected_result} != $result\n" ;} + +git_repo_4part="-~<:project:1" +expected_result=" ~:project:1" +result="" +printf "%-8s: %-80s : %-75s " "test" "byref_translate_from_git_repo_4part2ccm_4part" "${git_repo_4part} -> $expected_result" +byref_translate_from_git_repo_4part2ccm_4part "$git_repo_4part" result +[[ "$result" == "${expected_result}" ]] && { printf "%10s\n" "SUCCESS" ; }|| { printf " FAILED: ${expected_result} != $result\n" ;} + +ccm_4part="~ :project:1" +expected_result="~-:project:1" +result="" +printf "%-8s: %-80s : %-75s " "test" "byref_translate_from_ccm_4part2git_repo_4part" "${ccm_4part} -> $expected_result" +byref_translate_from_ccm_4part2git_repo_4part "${ccm_4part}" result +[[ "$result" == "${expected_result}" ]] && { printf "%10s\n" "SUCCESS" ; }|| { printf " FAILED: ${expected_result} != $result\n" ;} + diff --git a/_ccm-functions.sh b/_ccm-functions.sh new file mode 100644 index 0000000..67f4c43 --- /dev/null +++ b/_ccm-functions.sh @@ -0,0 +1,280 @@ +#!/usr/bin/env bash + +export regex_ccm4part="^(.+)${ccm_delim}(.+):(.+):(.+)$" + +function byref_translate_from_ccm_name2git_repo() { + if [[ -z ${1} ]]; then + echo "${FUNCNAME[0]}: Parameter 1 - by value - cannot be empty" && exit 1 + else + local _fromString=${1} + fi + if [[ -z ${2} ]]; then + echo "${FUNCNAME[0]}: Parameter 2 - as ref - cannot be empty" && exit 1 + else + local -n _toString=${2} + fi + _toString="${_fromString}" + _toString="${_toString//ener Tam/ener-Tam}" + _toString="${_toString// /-}" + _toString="${_toString//&/-}" + _toString="${_toString//#/-}" + _toString="${_toString//(/-}" + _toString="${_toString//)/-}" + _toString="${_toString//ü/ue}" + _toString="${_toString//ä/ae}" + _toString="${_toString//æ/ae}" + _toString="${_toString//å/aa}" + _toString="${_toString//ö/oe}" + _toString="${_toString//ø/oe}" + _toString="${_toString//,/-}" +} + +function byref_translate_from_git_repo2ccm_name_query() { + if [[ -z ${1} ]]; then + echo "${FUNCNAME[0]}: Parameter 1 - by value - cannot be empty" && exit 1 + else + local _fromString=${1} + fi + if [[ -z ${2} ]]; then + echo "${FUNCNAME[0]}: Parameter 2 - by ref - cannot be empty" && exit 1 + else + local -n _toString=${2} + fi + _toString="${_fromString}" + _toString="${_toString//ener-Tam/ener Tam}" + _toString="${_toString//-/?}" + _toString="${_toString//ue/??}" + _toString="${_toString//ae/??}" + _toString="${_toString//aa/??}" + _toString="${_toString//oe/??}" +} + +function byref_translate_from_ccm_version2git_tag() { +# https://stackoverflow.com/questions/26382234/what-names-are-valid-git-tags +#Disallowed characters: '~', '^', ':', '', '?', '[', '*' +#Disallowed sequences ".." and "@{" +#Disallowed Also tag cannot contain " " (whitespace) + if [[ -z ${1} ]]; then + echo "${FUNCNAME[0]}: Parameter 1 - by value - cannot be empty" && exit 1 + else + local _fromString=${1} + fi + if [[ -z ${2} ]]; then + echo "${FUNCNAME[0]}: Parameter 2 - as ref - cannot be empty" && exit 1 + else + local -n _toString=${2} + fi + _toString="${_fromString// /-}" + _toString="${_toString//&/-}" + _toString="${_toString//#/-}" + _toString="${_toString//(/-}" + _toString="${_toString//)/-}" +} + +function byref_translate_from_git_tag2ccm_version_query() { + if [[ -z ${1} ]]; then + echo "${FUNCNAME[0]}: Parameter 1 - by value - cannot be empty" && exit 1 + else + local _fromString=${1} + fi + if [[ -z ${2} ]]; then + echo "${FUNCNAME[0]}: Parameter 2 - by ref - cannot be empty" && exit 1 + else + local -n _toString=${2} + fi + _toString="${_fromString//-/?}" +} + + +function byref_translate_from_ccm_name_instance_query2ccm_name() { + if [[ -z ${1} ]]; then + echo "${FUNCNAME[0]}: Parameter 1 - by value - cannot be empty" && exit 1 + else + local _fromString=${1} + fi + if [[ -z ${2} ]]; then + echo "${FUNCNAME[0]}: Parameter 2 - by value - cannot be empty" && exit 1 + else + local _instance=${2} + fi + if [[ -z ${3} ]]; then + echo "${FUNCNAME[0]}: Parameter 3 - as ref - cannot be empty" && exit 1 + else + local -n _toString=${3} + fi + + local _query_string="name match '$_fromString' and type='project' and instance='${_instance}'" + local _found_project_name_instances=$(ccm query "${_query_string}" -u -f "%name:%instance" | /usr/bin/sort -u | wc -l) + if [[ _found_project_name_instances -eq 0 ]]; then + echo "ERROR: I found no projects with similar ? query name output gave foo and boo " + echo "$_query_string" + return 1 + fi + if [[ _found_project_name_instances -gt 1 ]]; then + echo "ERROR: I found two or more projects with similar ? query name output -oo gave foo and boo" + ccm query "${_query_string}" -u -f "%name:%instance" + return 1 + fi + _toString=$(ccm query "${_query_string}" -u -f "%name" | /usr/bin/sort -u ) +} + +function byref_translate_from_git_repo2ccm_name() { + if [[ -z ${1:-} ]]; then + echo "${FUNCNAME[0]}: Parameter 1 - by value - cannot be empty" && exit 1 + else + local _fromString=${1} + fi + if [[ -z ${2:-} ]]; then + echo "${FUNCNAME[0]}: Parameter 2 - by value - cannot be empty" && exit 1 + else + local _instance=${2} + fi + if [[ -z ${3:-} ]]; then + echo "${FUNCNAME[0]}: Parameter 3 - by ref - cannot be empty" && exit 1 + else + local -n _toString=${3} + fi + + local _git_repo_name=$_fromString + local _ccm_instance=$_instance + local _ccm_query_name="" + byref_translate_from_git_repo2ccm_name_query "${_git_repo_name}" _ccm_query_name + local _query_result="" + byref_translate_from_ccm_name_instance_query2ccm_name "${_ccm_query_name}" "${_instance}" _query_result + _toString=$_query_result +} + +function byref_translate_from_git_repo_4part2ccm_4part() { + if [[ -z ${1} ]]; then + echo "${FUNCNAME[0]}: Parameter 1 - by value - cannot be empty" && exit 1 + else + local _fromString=${1} + fi + if [[ -z ${2} ]]; then + echo "${FUNCNAME[0]}: Parameter 2 - by ref - cannot be empty" && exit 1 + else + local -n _toString=${2} + fi + + [[ "${_fromString:-}" =~ ${regex_ccm4part} ]] || { + echo "4part does not comply" + return 1 + } + local _name=${BASH_REMATCH[1]} + local _version=${BASH_REMATCH[2]} + local _type=${BASH_REMATCH[3]} + local _instance=${BASH_REMATCH[4]} + + local _ccm_query_name="" + byref_translate_from_git_repo2ccm_name_query "${_name}" _ccm_query_name + + local _ccm_query_version="" + byref_translate_from_git_tag2ccm_version_query "${_version}" _ccm_query_version + + local _ccm_query_4name="$_ccm_query_name-$_ccm_query_version:$_type:$_instance" + + local _query_result="" + byref_translate_from_ccm_4part_query2ccm_4part "${_ccm_query_4name}" _query_result + _toString=$_query_result +} + +function byref_translate_from_ccm_4part2git_repo_4part() { + if [[ -z ${1} ]]; then + echo "${FUNCNAME[0]}: Parameter 1 - by value - cannot be empty" && exit 1 + else + local _fromString=${1} + fi + if [[ -z ${2} ]]; then + echo "${FUNCNAME[0]}: Parameter 2 - by ref - cannot be empty" && exit 1 + else + local -n _toString=${2} + fi + + [[ "${_fromString:-}" =~ ${regex_ccm4part} ]] || { + echo "4part does not comply" + return 1 + } + local _name=${BASH_REMATCH[1]} + local _version=${BASH_REMATCH[2]} + local _type=${BASH_REMATCH[3]} + local _instance=${BASH_REMATCH[4]} + + local _git_repo="" + byref_translate_from_ccm_name2git_repo "${_name}" _git_repo + + local _git_tag="" + byref_translate_from_ccm_version2git_tag "${_version}" _git_tag + + _toString="${_git_repo}-${_git_tag}:$_type:$_instance" + +} + +function byref_translate_from_ccm_4part_query2ccm_4part() { + if [[ -z ${1} ]]; then + echo "Parameter 1 - as ref - cannot be empty" && return 1 + else + local _ccm4part_query=${1} + fi + if [[ -z ${2} ]]; then + echo "Parameter 2 - as ref - cannot be empty" && return 1 + else + local -n _ccm4part=${2} + fi + + [[ "${_ccm4part_query:-}" =~ ${regex_ccm4part} ]] || { + echo "4part does not comply" + return 1 + } + local name=${BASH_REMATCH[1]} + local version=${BASH_REMATCH[2]} + local type=${BASH_REMATCH[3]} + local instance=${BASH_REMATCH[4]} + + _ccm4part=$(ccm query "name match '$name' and version match '$version' and type='$type' and instance='${instance}'" -u -f "%objectname") +} + +function find_n_set_baseline_obj_attrs_from_project(){ + local ccm_project_4part=$1 + local verbose="true" + [[ ${2:-} == "verbose_false" ]] && local verbose="false" + + [[ ${ccm_project_4part} =~ ${regex_ccm4part} ]] || exit 1 + proj_name=${BASH_REMATCH[1]} + proj_version=${BASH_REMATCH[2]} + proj_instance=${BASH_REMATCH[4]} + + project_release=$(ccm properties -f "%release" "${ccm_project_4part}") || return $? + if [[ "$project_release" == "" ]]; then + project_release="void" + release_query="" + else + release_query=" and release='${project_release}'" + fi + + # Find the baseline object of the project with the same release as the project itself + ccm_baseline_obj_and_status_release_this=$(ccm query "has_project_in_baseline('${ccm_project_4part}') ${release_query}" -sby create_time -u -f "%objectname@@@%status@@@%release" | head -1 ) + regex_baseline_attr='^(.+)@@@(.+)@@@(.+)$' + if [[ "${ccm_baseline_obj_and_status_release_this:-}" == "" ]]; then + # No baseline found with primary release tag .. See if other baseline objects are connected ( eg. list any Baseline Object and accept the first ) + ccm_baseline_obj_and_status_release_this=$(ccm query "has_project_in_baseline('${ccm_project_4part}')" -sby create_time -u -f "%objectname@@@%status@@@%release" | head -1 ) + if [[ "${ccm_baseline_obj_and_status_release_this:-}" == "" ]]; then + if [[ "${verbose:-}" == "true" ]]; then + echo "NOTE: No related Baseline Object not found at all: ${ccm_project_4part}" >&2 + fi + else + [[ ${ccm_baseline_obj_and_status_release_this} =~ ${regex_baseline_attr} ]] || exit 1 + ccm_baseline_obj=${BASH_REMATCH[1]} + ccm_baseline_status=${BASH_REMATCH[2]} + ccm_baseline_release=${BASH_REMATCH[3]} + if [[ ${verbose:-} == "true" ]]; then + echo "NOTE: release diff found.. ${ccm_project_4part} / ${project_release} <=> ${ccm_baseline_release} / ${ccm_baseline_obj} - accepted" >&2 + fi + fi + else + [[ ${ccm_baseline_obj_and_status_release_this} =~ ${regex_baseline_attr} ]] || exit 1 + ccm_baseline_obj=${BASH_REMATCH[1]} + ccm_baseline_status=${BASH_REMATCH[2]} + ccm_baseline_release=${BASH_REMATCH[3]} + fi +} + diff --git a/_ccm-start-stop-functions.sh b/_ccm-start-stop-functions.sh new file mode 100644 index 0000000..8efb2fb --- /dev/null +++ b/_ccm-start-stop-functions.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +[[ ${debug:-} == true ]] && set -x + +[[ ${ccm_server_db_path:-} == "" ]] && { + export ccm_server_db_path=//data/ccmdb + echo "Setting the default data path to: ccm_server_db_path=$ccm_server_db_path" + echo " - you can set in your bashrc or similar to overwrite" +} + +[[ ${ccm_server:-} == "" ]] && { + echo "ccm_server is not set.." + echo " - you can set in your bashrc or export it explicitly" + return 1 +} + +function check_ccm_bin_n_set_CCM_HOME() { + echo "Checking if the 'ccm' bin is found: " + export CCM_HOME=$(dirname $(dirname $(which ccm 2> /dev/null ))) || { + echo "ERROR: cannot file 'ccm' in PATH" + return 1 + } + +} + +function check_param() { + if [[ $1 == "" ]]; then + echo "Please set database as parameter 1" + return 2 + fi +} + +function check_user_env_var() { + if [[ ${CCM_USER:-} != "" ]]; then + export ccm_user_option="-n" + echo "Setting ccm user to ${CCM_USER}" + else + echo "Using default user as CM/Synergy user" + fi +} + + +function check_db_started_already() { + echo "Checking if session already started" + if [[ ${ccm_db:-} == "" && ${CCM_ADDR:-} == "" ]]; then + echo "ccm_db and CCM_ADDR is empty - assume ok" + return 0 + fi + if ccm status | grep "No sessions found"; then + return 0 + fi + + if ccm status | grep -A 1 current; then + echo "ERROR: Already have a session running ... " + return 3 + fi +} + +function ccm-start() { + { + check_param $1 || return $? + check_ccm_bin_n_set_CCM_HOME || return $? + check_db_started_already || return $? + check_user_env_var + CCM_ADDR=$(ccm start ${ccm_user_option:-} ${CCM_USER:-} -m -d ${ccm_server_db_path}/${1} -s ${ccm_server} -q) + export CCM_ADDR + echo "CCM_ADDR=${CCM_ADDR}" + + ccm_db=$1 + export ccm_db + + ccm_delim=$(ccm delimiter) + export ccm_delim + echo "ccm_delim=${ccm_delim}" + + ccm_dcm_dbid=$(ccm dcm -show -dbid) + export ccm_dcm_dbid + echo "ccm_dcm_dbid=${ccm_dcm_dbid}" + + } || { unset ccm_db ; return $?; } +} + +function check_db_started_for_stop() { + if [[ "${CCM_ADDR:-}" == "" ]]; then + echo "CCM_ADDR is empty - exit with noop" + unset ccm_db + return 0 + fi + if ccm status | grep "No sessions found"; then + return 0 + fi + if ccm status | grep -A 1 current; then + return 1 + fi +} + +function ccm-stop() { + { + ccm stop || true + echo "unset ccm variables: CCM_HOME CCM_ADDR ccm_db ccm_delim ccm_dcm_dbid" + unset CCM_HOME + unset CCM_ADDR + unset ccm_db + unset ccm_delim + unset ccm_dcm_dbid + } || return $? + +} + +if [[ $- == *i* ]]; then + echo "Synergy is now setup as functions in your bash shell.." + echo "Use:" + echo "- ccm-start " + echo "- ccm-stop" +fi + +#ccm-start $* + +set +x diff --git a/_git-functions.sh b/_git-functions.sh new file mode 100644 index 0000000..cd51a25 --- /dev/null +++ b/_git-functions.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash + +function git_rename_annotated_tag_remote_n_locally(){ + local git_old_tag=$1 + local git_new_tag=$2 + + # reset the committer to get the correct set for the commiting the tag. There is no author of the tag + export GIT_AUTHOR_DATE=$(git tag -l --format="%(taggerdate:iso8601)" ${git_old_tag} | awk -F" " '{print $1 " " $2}') && [[ -z ${GIT_AUTHOR_DATE} ]] && return 1 + export GIT_COMMITTER_DATE=${GIT_AUTHOR_DATE} + export GIT_COMMITTER_NAME=$(git tag -l --format="%(taggername)" ${git_old_tag} ) && [[ -z ${GIT_COMMITTER_NAME} ]] && return 1 + export GIT_COMMITTER_EMAIL=$(git tag -l --format="%(taggeremail)" ${git_old_tag} ) && [[ -z ${GIT_COMMITTER_EMAIL} ]] && return 1 + + echo "Get tag content of: ${git_old_tag}" + git tag -l --format '%(contents)' ${git_old_tag} > ./tag_meta_data.txt + echo "git commit content of ${git_old_tag}" + echo "git tag ${git_new_tag} based on ${git_old_tag}" + git tag -a -F ./tag_meta_data.txt ${git_new_tag} ${git_old_tag}^{} || return 1 + rm -f ./tag_meta_data.txt + git tag --delete ${git_old_tag} || return 1 +} + +function git_delete_tag_on_remote_repos(){ + local git_remote=$1 + local git_tag=$2 + git push ${git_remote} --delete ${git_tag} +} + +function git_create_tag_on_remote_repos(){ + local git_remote=$1 + local git_tag=$2 + git push ${git_remote} ${git_tag} +} + +function git_resolve_tags_wstatus() { + local repo_submodule=$1 + local repo_submodule_rev=$2 + export repo_submodule_rev_wcomponent_wstatus=$(git tag | grep ${repo_submodule}/.*/${repo_submodule_rev}_rel$ || grep_exit=$? ) + # Look then for the "pub" tag + if [[ ${repo_submodule_rev_wcomponent_wstatus} == "" ]]; then + export repo_submodule_rev_wcomponent_wstatus=$(git tag | grep ${repo_submodule}/.*/${repo_submodule_rev}_pub$ || grep_exit=$? ) + else + return 0 + fi + # Accept what is there of remaining + if [[ ${repo_submodule_rev_wcomponent_wstatus} == "" ]]; then + export repo_submodule_rev_wcomponent_wstatus=$(git tag | grep ${repo_submodule}/.*/${repo_submodule_rev}_[dprtis][eueenq][lblsta]$ || grep_exit=$? ) + else + return 0 + fi +} + +function git_initialize_lfs_n_settings() { + echo "Installing Git LFS for the repo" + git lfs install + if [[ ! $( git config --get lfs.locksverify ) ]] ; then + git config --add --local 'lfs.locksverify' false + fi + if [[ ! $( git config --get lfs.contenttype ) ]] ; then + git config --add --local lfs.contenttype 0 + fi +} + +function git_set_execute_bit_in_index_of_extensions() { + # PRE: you are in the repo + exit_code=0 + echo "Based on file extension - set execute bit in the repo on following files:" + git ls-files | grep -ie '.*\.exe$' -ie '.*\.sh$' -ie '.*\.pl$' || exit_code=$? + if [[ ${exit_code} -eq 0 ]] ; then + git ls-files | grep -ie '.*\.exe$' -ie '.*\.sh$' -ie '.*\.pl$' | xargs --no-run-if-empty -d '\n' git update-index --add --chmod=+x + git ls-files | grep -ie '.*\.exe$' -ie '.*\.sh$' -ie '.*\.pl$' | xargs --no-run-if-empty -d '\n' chmod +x + elif [[ ${exit_code} -eq 1 ]] ; then + echo "No files found" + else + echo "ERROR occured" + exit ${exit_code} + fi + echo "Done" +} + +function _git_set_execute_bit(){ + echo Making ${git_ls_file} executable + git update-index --add --chmod=+x ${git_ls_file} + chmod +x ${git_ls_file} +} + +function git_set_execute_bit_in_index_of_unix_tool_file_executable() { + # PRE: you are in the repo + set +x + echo "Based on file reporting 'executable' or 'interpreter' - set execute bit in the repo on following files:" + IFS=$'\r\n' + for git_ls_file in `git ls-files`; do + if file ${git_ls_file} |grep executable 2>&1 > /dev/null; then + _git_set_execute_bit + elif file ${git_ls_file} |grep interpreter 2>&1 > /dev/null; then + _git_set_execute_bit + fi + done + [[ ${debug:-} == "true" ]] && set -x + echo "Done" +} + +function git_find_n_fill_empty_dirs_gitignore() { + echo "Fill empty directories with .gitignore" + file_empty_dirs_tmp="empty_dirs.tmp" + local exit_code=0 + /usr/bin/find . -mindepth 1 -type d -empty | grep -v '\./\.git/' > ${file_empty_dirs_tmp} || exit_code=$? + if [[ ${exit_code} -eq 0 ]] ; then + # grep did give an output - hence there are empty directories + IFS=$'\r\n' + while read empty_dir; do + echo ${empty_dir} + cp ${script_dir}/emptydir.gitignore ${empty_dir}/.gitignore + done < ${file_empty_dirs_tmp} || exit 1 + unset IFS + elif [[ ${exit_code} -eq 1 ]] ; then + # grep did not give an output - hence there are no empty directories + echo "INFO: no empty directories where found" + else + echo "ERROR occured" + exit ${exit_code} + fi + rm -f ${file_empty_dirs_tmp} + echo "Done" +} + +function git_copy_tag(){ + local source_tag="$1" + local target_tag="$2" + local GIT_COMMITTER_NAME="$(git tag -l --format=\"%\(taggername\)\" \"$source_tag\" )" + local GIT_COMMITTER_EMAIL="$(git tag -l --format=\"%\(taggeremail\)\" \"$source_tag\" )" + local GIT_COMMITTER_DATE="$(git tag -l --format=\"%\(taggerdate:iso8601\)\" \"$source_tag\" )" + git tag \ + "$target_tag" \ + "$(git tag -l --format=\"%\(*objectname\)\" \"$source_tag\" )" \ + -a \ + -m "$(git tag -l --format=\"%\(contents\)\" \$source_tag\" )" +} \ No newline at end of file diff --git a/build.gradle b/build.gradle index c6d4c3b..77c3980 100644 --- a/build.gradle +++ b/build.gradle @@ -19,52 +19,45 @@ apply plugin: 'maven-publish' repositories { mavenCentral() maven { - url 'https://repo.jenkins-ci.org/public/' - } - maven { - // org.jfrog.artifactory.client:artifactory-java-client-services:1.2.2 is published here - url 'https://releases.jfrog.io/artifactory/oss-releases/' + url = 'https://repo.jenkins-ci.org/public/' } } dependencies { - // COOL - compile fileTree(dir: 'libs', include: ['*.jar']) - compile "net.praqma:praqmajutils:1.0.0" - - // Other - compile 'org.slf4j:slf4j-api:1.7.25' - compile 'org.codehaus.groovy:groovy-all:2.4.12' - compile 'org.jfrog.artifactory.client:artifactory-java-client-services:1.2.2' - testCompile 'org.spockframework:spock-core:1.0-groovy-2.4' + implementation group: 'org.apache.groovy', name: 'groovy-all', version: '4.0.24' + implementation 'org.slf4j:slf4j-api:2.0.16' + implementation 'org.slf4j:slf4j-simple:2.0.16' + implementation('org.jfrog.artifactory.client:artifactory-java-client-services:2.19.1') { + exclude group: 'org.codehaus.groovy' + } + implementation fileTree('libs') { + include '*.jar' + } + implementation 'net.praqma:praqmajutils:1.0.0' + testImplementation group: 'org.spockframework', name: 'spock-core', version: '2.3-groovy-4.0' } // -------------------- // project configuration // -------------------- -project.group = 'com.praqma' -project.version = gitVersion() -project.description = 'Migration engine used to migrate from an arbitrary VCS to git' - -// -------------------- -// application plugin -// -------------------- - -project.mainClassName = 'togit.Run' - -startScripts { - doLast { - // The start scripts can't handle a project name starting with a number. - // They'd contain use of a parameter called '$2GIT_OPTS', in which the '$2' was problematic. - unixScript.text = unixScript.text.replaceAll('2GIT_OPTS', 'TOGIT_OPTS') - windowsScript.text = windowsScript.text.replaceAll('2GIT_OPTS', 'TOGIT_OPTS') +task uberjar(type: Jar) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + dependsOn('compileGroovy') + from files(sourceSets.main.output.classesDirs) + from configurations.runtimeClasspath.asFileTree.files.collect { zipTree(it) } + manifest { + attributes 'Main-Class': 'toGit.Run' } } -// -------------------- -// github plugin -// -------------------- +application { + mainClass = 'toGit.Run' +} + +test { + exclude 'toGit/functional/**' +} github { owner = 'Praqma' diff --git a/build/libs/2git-0.4.1.jar b/build/libs/2git-0.4.1.jar new file mode 100644 index 0000000..6616a88 Binary files /dev/null and b/build/libs/2git-0.4.1.jar differ diff --git a/ccm-baseline-history-get-root.sh b/ccm-baseline-history-get-root.sh new file mode 100644 index 0000000..b7d3cbe --- /dev/null +++ b/ccm-baseline-history-get-root.sh @@ -0,0 +1,48 @@ +#!/bin/bash +[[ ${debug:-} == true ]] && set -x +set -u +set -e + +source $(dirname $0)/_ccm-functions.sh || source ./_ccm-functions.sh + +if [[ ${1} == "" ]]; then + printf "project parameter 1 is empty - skip" 1>&2 + exit 0 +fi +BASELINE_PROJECT_tmp="$1" +BASELINE_PROJECT="$1" + + +printf "Processing ${BASELINE_PROJECT}\n" 1>&2 + +until [[ "${BASELINE_PROJECT:-}" == "" ]] ; do + this_project4part="${BASELINE_PROJECT}" + + this_project_name=$(echo ${BASELINE_PROJECT} | awk -F"${ccm_delim}" '{print $1}') + + query="is_baseline_project_of('${BASELINE_PROJECT}')" + BASELINE_PROJECT=$(ccm query "is_baseline_project_of('${BASELINE_PROJECT}')" -u -f "%objectname") || BASELINE_PROJECT="" + baseline_name=`printf "${BASELINE_PROJECT:-}" | awk -F"${ccm_delim}" '{print $1}'` + if [[ "${baseline_name:-}" != "" && "${baseline_name}" != "${this_project_name}" ]]; then + printf "Stop traversing - name changed: '${this_project_name}' -> '$baseline_name'\n\n" 1>&2 + printf "Get sucessors with '${this_project_name}' of baseline_project '${BASELINE_PROJECT}' (different name) as well\n\n" 1>&2 + echo "${this_project4part}" + ccm query "has_baseline_project('${BASELINE_PROJECT}') and name='${this_project_name}' " -u -f "%objectname" + exit + fi + if [[ "${BASELINE_PROJECT:-}" != "" ]] ; then + printf "${BASELINE_PROJECT} -> " 1>&2 + else + printf " - baseline empty\n\n" 1>&2 + break + fi +done +project_status=$(ccm attr -show status "${this_project4part}") +case "$project_status" in + integrate|released|sqa|test) + printf "${this_project4part}" + ;; + *) + printf "Project $this_project4part is in state: $project_status - Skip - do not list" 1>&2 + ;; +esac diff --git a/ccm-baseline-history.sh b/ccm-baseline-history.sh new file mode 100644 index 0000000..0e1d1a4 --- /dev/null +++ b/ccm-baseline-history.sh @@ -0,0 +1,142 @@ +#!/bin/bash + +set -euo pipefail + +[[ ${debug:-} == "true" ]] && set -x + + +use_wildcard="" # * + +# Load functions +source $(dirname $0)/_ccm-functions.sh || source ./_ccm-functions.sh + +find_project_baseline_to_convert(){ + local CURRENT_PROJECT=$1 + local inherited_string_local=$2 + + if [[ "${use_wildcard}" == "*" ]]; then + proj_name=$(printf "${CURRENT_PROJECT}" | awk -F"${ccm_delim}|:" '{print $1}') + proj_version=$(printf "${CURRENT_PROJECT}" | awk -F"${ccm_delim}|:" '{print $2}') + proj_instance=$(printf "${CURRENT_PROJECT}" | awk -F"${ccm_delim}|:" '{print $4}') + query="has_baseline_project(name match '${proj_name}*' and version='${proj_version}' and type='project' and instance='${proj_instance}') and ( status='integrate' or status='test' or status='sqa' or status='released' )" + else + [[ ${CURRENT_PROJECT:-} =~ ${regex_ccm4part} ]] || { + echo "4part does not comply" + return 1 + } + local proj_name=${BASH_REMATCH[1]} + local version=${BASH_REMATCH[2]} + local type=${BASH_REMATCH[3]} + local instance=${BASH_REMATCH[4]} + + query="has_baseline_project('${CURRENT_PROJECT}') and project='$proj_name' and ( status='integrate' or status='test' or status='sqa' or status='released' )" + fi + + # All status versions + [[ ${debug:-} == "true" ]] && { + printf "_________________________\n%s\n" "$(ccm query "${query}" -u -f "%objectname")" >&2 + } + IFS=$'\r\n' + for SUCCESSOR_PROJECT in $(ccm query "${query}" -u -f "%objectname") ; do + local inherited_string="${inherited_string_local} -> ${CURRENT_PROJECT}" + [[ ${debug:-} == "true" ]] && printf "${inherited_string}\n" >&2 + if [[ $(ccm properties -f %ccm2git_migrate "${SUCCESSOR_PROJECT}" ) == "FALSE" ]]; then + echo "SKIP: ${SUCCESSOR_PROJECT} ccm2git_migrate=FALSE - continue" >&2 + continue # Next if already for some odd reason exists - seen in firebird~BES-SW-0906-1.8:project:2 + fi + if [[ `grep "$SUCCESSOR_PROJECT@@@$CURRENT_PROJECT" ${projects_file}` ]]; then + echo "ALREADY include in project file - continue" >&2 + continue # Next if already for some odd reason exists - seen in firebird~BES-SW-0906-1.8:project:2 + fi + exit_code="0" + find_n_set_baseline_obj_attrs_from_project "${SUCCESSOR_PROJECT}" "verbose_true" || exit_code=$? + if [[ "${exit_code}" != "0" ]] ; then + echo "ERROR: Project not found: ${SUCCESSOR_PROJECT}" + exit ${exit_code} + fi + if [[ ${ccm_baseline_status:-} == "test_baseline-DONT_CARE" ]] ; then + # Figure out if the project is in use as as baseline in and other project + exit_code=0 + project_baseline_childs=$(ccm query "has_baseline_project('${SUCCESSOR_PROJECT}') and ( status='integrate' or status='test' or status='sqa' or status='released' )" -u -f "%objectname" | head -1 ) || exit_code=$? + if [[ $exit_code -eq 6 ]]; then + project_baseline_childs="" + elif [[ $exit_code -ne 0 ]]; then + echo "ERROR: something when wrong: exit code: $exit_code" + exit $exit_code + fi + if [[ "${project_baseline_childs:-}" != "" ]]; then + echo "ACCEPT: Related Baseline Object is in test status: ${SUCCESSOR_PROJECT}: ${ccm_baseline_obj_and_status_release_this} - but at least in use as baseline of project: ${project_baseline_childs}" >&2 + else + if [[ $(ccm finduse -all_projects "$(echo ${SUCCESSOR_PROJECT} )" | grep "Object is not used in scope." ) ]]; then + # not in use as sub project" + echo "SKIP: Related Baseline Object is in test status and is NOT in use as subproject: ${SUCCESSOR_PROJECT}: ${ccm_baseline_obj_and_status_release_this} - and is leaf in project baseline history" >&2 + continue + else + # in use + echo "ACCEPT: Related Baseline Object is in test status and is in use as subproject: ${SUCCESSOR_PROJECT}: ${ccm_baseline_obj_and_status_release_this} - even is leaf in project baseline history" >&2 + fi + fi + fi + regex_revision_contains_History='^.*\.History.*$' + if [[ ${proj_version:-} =~ ${regex_revision_contains_History} ]] ; then + exit_code=0 + project_baseline_childs=$(ccm query "has_baseline_project('$(echo ${SUCCESSOR_PROJECT} )') and ( status='integrate' or status='test' or status='sqa' or status='released' )" -u -f "%objectname" | head -1 ) || exit_code=$? + if [[ $exit_code -eq 6 ]]; then + project_baseline_childs="" + elif [[ $exit_code -ne 0 ]]; then + echo "ERROR: something when wrong: exit code: $exit_code" + exit $exit_code + fi + if [[ "${project_baseline_childs:-}" != "" ]]; then + echo "ACCEPT: Project revision contains 'History': ${SUCCESSOR_PROJECT}: ${ccm_baseline_obj_and_status_release_this} - but is in use as baseline of project: ${project_baseline_childs}" >&2 + else + if [[ $(ccm finduse -all_projects "$(echo ${SUCCESSOR_PROJECT} )" | grep "Object is not used in scope." ) ]]; then + # not in use as sub project" + echo "SKIP: Project revision contains 'History and is NOT in use as subproject: ${SUCCESSOR_PROJECT}: ${ccm_baseline_obj_and_status_release_this} - even is leaf in project baseline history" >&2 + continue + else + # in use + echo "ACCEPT: Project revision contains '.History', but is in use as a subproject: ${SUCCESSOR_PROJECT}: ${ccm_baseline_obj_and_status_release_this} - even is leaf in project baseline history" >&2 + fi + fi + fi + git_CURRENT_PROJECT="" + byref_translate_from_ccm_4part2git_repo_4part "${CURRENT_PROJECT}" git_CURRENT_PROJECT + + git_SUCCESSOR_PROJECT="" + byref_translate_from_ccm_4part2git_repo_4part "${SUCCESSOR_PROJECT}" git_SUCCESSOR_PROJECT + printf "${git_SUCCESSOR_PROJECT}@@@${git_CURRENT_PROJECT}@@@${SUCCESSOR_PROJECT}@@@${CURRENT_PROJECT}\n" >> ${projects_file} + find_project_baseline_to_convert "${SUCCESSOR_PROJECT}" "${inherited_string}" + done +} + +## MAIN ## +git_BASELINE_PROJECT="$1" + +export projects_file="./projects.txt" +#rm -f ${projects_file} +if [ "${use_cached_project_list:-}X" == "trueX" ]; then + if [ -e ${projects_file} ] ; then + cat ${projects_file} + exit 0 + fi +fi + +ccm_BASELINE_PROJECT="" +byref_translate_from_git_repo_4part2ccm_4part "${git_BASELINE_PROJECT}" ccm_BASELINE_PROJECT + + +checked_version=$(ccm properties -f %version "$ccm_BASELINE_PROJECT" ) || { + exit_code=$? + echo "Project: $ccm_BASELINE_PROJECT does not exit - exit 1" + exit $exit_code +} + +init_project_name=$(printf "${ccm_BASELINE_PROJECT}" | awk -F"${ccm_delim}" '{print $1}') +instance=$(printf "${ccm_BASELINE_PROJECT}" | awk -F"${ccm_delim}|:" '{print $4}' ) + +inherited_string="${ccm_BASELINE_PROJECT}" +echo "$git_BASELINE_PROJECT@@@${init_project_name}${ccm_delim}init:project:${instance}@@@$ccm_BASELINE_PROJECT@@@${init_project_name}${ccm_delim}init:project:${instance}" > ${projects_file} + +find_project_baseline_to_convert "${ccm_BASELINE_PROJECT}" "${inherited_string}" +cat ${projects_file} diff --git a/ccm-convert-flat2submodules-existingRepoHistory.sh b/ccm-convert-flat2submodules-existingRepoHistory.sh new file mode 100644 index 0000000..77a0c7f --- /dev/null +++ b/ccm-convert-flat2submodules-existingRepoHistory.sh @@ -0,0 +1,725 @@ +#!/usr/bin/env bash +[[ ${debug:-} == "true" ]] && set -x +[[ ${run_local:-} == "true" ]] && push_remote="false" +set -u +set -e +set -o pipefail + +# Load functions +source $(dirname $0)/_ccm-functions.sh || source ./_ccm-functions.sh +source $(dirname $0)/_git-functions.sh || source ./_git-functions.sh + +execution_root_directory=$(pwd) + +# parameter $1 is the project list file generated by the baseline_history.sh script +export repo_name=${1} + +export repo_init_tag=${2} +export repo_submodules=${3} +[[ ${submodule_update_mode:-} == "" ]] && export submodule_update_mode="directory" # or update-index which is old style +[[ ${push_tags_in_submodules:-} == "" ]] && export push_tags_in_submodules="false" +[[ "${execute_mode:-}" == "" ]] && export execute_mode="normal" + +export gitrepo_project_original=${4} +export project_instance=${5} +export gitignore_path_n_files=${6} # :@:.. +export gitattributes_path_n_files=${7:-} # :@:.. + +export ccm_name="" +byref_translate_from_git_repo2ccm_name "${repo_name}" "$project_instance" ccm_name + +declare -A repo_submodules_map +if [[ $(echo "${repo_submodules}" | grep "," ) ]]; then + IFS="," +else + IFS=" " +fi +for repo_submodule_from_param in $(echo "${repo_submodules}"); do + repo_submodule_raw_name=$(echo ${repo_submodule_from_param} | awk -F ":" '{print $1}') + repo_submodules_map["${repo_submodule_raw_name}"]="${repo_submodule_raw_name}" +done +unset repo_submodule_raw_name +unset repo_submodule_from_map +unset IFS + +#export project_revisions=`cat ${1}` + +if [[ "${BITBUCKET_PROD_USERNAME:-}" != "" && "${BITBUCKET_PROD_PASSWORD:-}" != "" ]]; then + http_remote_credentials="${BITBUCKET_PROD_USERNAME}:${BITBUCKET_PROD_PASSWORD}@" +else + http_remote_credentials="" +fi + +export git_ssh_remote=ssh://git@${git_server_path}/${repo_name}.git +export git_ssh_remote_orig=ssh://git@${git_server_path}/${repo_name}_orig.git +export git_https_remote=$(echo ${git_ssh_remote} | sed -e "s/ssh:\/\/git@/https:\/\/${http_remote_credentials}/" -e 's/7999/7990\/scm/' | sed -e 's/ssh-//' | sed -e 's/:7990//') +export git_https_remote_orig=$(echo ${git_ssh_remote_orig} | sed -e "s/ssh:\/\/git@/https:\/\/${http_remote_credentials}/" -e 's/7999/7990\/scm/' | sed -e 's/ssh-//' | sed -e 's/:7990//' ) +export git_remote_to_use=${git_https_remote} +export git_remote_to_use_orig=${git_https_remote_orig} +echo "INFO: Use remote : ${git_remote_to_use} and ${git_remote_to_use_orig}" + +function resolve_git_common_target_repo { + local dir_name_from_git_common_target_repo=$(dirname ${git_common_target_repo} ) + if [[ $dir_name_from_git_common_target_repo == "." ]]; then + echo "The git_common_target_repo does not include relative prefix - use it directly - i.e. same project - replace repo " + local server_base_url=$(dirname "$git_remote_to_use") + export https_remote_common=${server_base_url}/${git_common_target_repo} + else + echo "The git_common_target_repo contains relative prefix - use it directly with / - i.e. same project" + local server_base_url=$(dirname $(dirname "$git_remote_to_use")) + export https_remote_common="${server_base_url}/${git_common_target_repo}" + fi +} + +if [[ "${git_common_target_repo:-}" != "" ]]; then + + resolve_git_common_target_repo + + echo "INFO: Configured to common repo: ${git_common_target_repo} : ${https_remote_common} " + echo " - consider to enable 'push_to_remote_during_conversion=true'" + echo " to avoid false failed fetch of tags in case of history rewrite" +fi + +function convert_revision(){ + local root_dir=$(pwd) + local repo_convert_rev_tag=$1 + + local tag_to_convert=$(git tag | grep "${repo_name}/.*/${repo_convert_rev_tag}$" || grep_ext_value=$?) + + set +x + if [[ "${tag_to_convert}" == "" ]] ; then + echo "============================================================================" + echo " BEGIN: ${repo_convert_rev_tag}" + echo "============================================================================" + [[ ${debug:-} == "true" ]] && set -x + else + echo "=====================================================================================================" + echo " Already done - skip: ${repo_convert_rev_tag} -> ${tag_to_convert}" + echo "=====================================================================================================" + [[ ${debug:-} == "true" ]] && set -x + return + fi + local ccm_repo_convert_rev_tag=${repo_convert_rev_tag:: -4} + + local exit_code="0" + git_4part="${repo_name}${ccm_delim}${ccm_repo_convert_rev_tag}:project:${project_instance}" + + ccm_4part="" + byref_translate_from_git_repo_4part2ccm_4part "${git_4part}" ccm_4part + + find_n_set_baseline_obj_attrs_from_project "${ccm_4part}" "verbose_false" || exit_code=$? + if [[ "${exit_code}" != "0" ]] ; then + echo "ERROR: Project not found: ${ccm_4part}" + exit ${exit_code} + fi + local ccm_release=$(echo ${project_release} | cut -d "/" -f 2) # inherited from function find_n_set_baseline_obj_attrs_from_project + + [[ "${ccm_release:-}" == "x" ]] && ( echo "Release is empty!!" && exit 1) + [[ "${ccm_release:-}" == "" ]] && ( echo "Release is !!" && exit 1) # if the release is it is rewritten to void as BitBucket does not allow <> chars in tags + + #NOTE: The next line is suppressing the support for having a baseline project with a different name than is being converted: ( and name='${repo_name}' ) + local ccm_baseline_from_tag_info="$(ccm query "is_baseline_project_of('${ccm_4part}') and name='${ccm_name}'" \ + -u -f "%version" )" || return 1 + + + if [[ "${ccm_baseline_from_tag_info}" != "" ]] ; then + # prefer released if found + baseline_from_tag_info="" + byref_translate_from_ccm_version2git_tag "$ccm_baseline_from_tag_info" baseline_from_tag_info + + local repo_baseline_rev_tag_wcomponent_wstatus=$(git tag | grep ^${repo_name}/.*/${baseline_from_tag_info}_rel$ || grep_ext_value=$? ) + if [[ "${repo_baseline_rev_tag_wcomponent_wstatus}" == "" ]]; then + local repo_baseline_rev_tag_wcomponent_wstatus=$(git tag | grep ^${repo_name}/.*/${baseline_from_tag_info}_[dprtis][eueenq][lblsta]$ || grep_ext_value=$? ) + tag_amounts=$(echo ${repo_baseline_rev_tag_wcomponent_wstatus} | wc -l ) + [[ ${tag_amounts} -gt 1 ]] && { printf "ERROR: More than one tag (${tag_amounts}) found in repo_baseline_rev_tag_wcomponent_wstatus\n${repo_baseline_rev_tag_wcomponent_wstatus}\n" && return 1 ;} + fi + if [[ "${repo_baseline_rev_tag_wcomponent_wstatus}" == "" ]] ; then + #find the original tag and convert it first + local repo_baseline_orig_tag_wstatus=$(git tag | grep "^${baseline_from_tag_info}_[dprtis][eueenq][lblsta]$" || grep_ext_value=$?) + if [[ "${repo_baseline_orig_tag_wstatus}" != "" ]]; then + echo "INFO: RECURSIVE action - Wait with ${ccm_repo_convert_rev_tag} / baseline_from_tag_info_wstatus is empty hence the order of tags of the single commit is listed in the wrong order of tags on same commit" + convert_revision ${repo_baseline_orig_tag_wstatus} + # reset it to make sure the recursive have not reset it + local repo_baseline_rev_tag_wcomponent_wstatus=$(git tag | grep ^${repo_name}/.*/${baseline_from_tag_info}_[dprtis][eueenq][lblsta]$ || grep_ext_value=$? ) + set +x + echo "============================================================================" + echo " CONTINUE: ${repo_convert_rev_tag}" + echo "============================================================================" + [[ ${debug:-} == "true" ]] && set -x + else + echo "ERROR: Dont know why we ended here - something is not right!!" + echo "The baseline tag that is needed for the conversion of tag: ${repo_convert_rev_tag} cannot even find the tag unconverted: ${baseline_from_tag_info}" + return 1 + fi + fi + else + local repo_baseline_rev_tag_wcomponent_wstatus="${repo_name}/${repo_init_tag}/${repo_init_tag}" + fi + if [[ -f ${execution_root_directory}/${ccm_db}_gitfiles/baseline_history_rewrite.txt ]]; then + # lookup in the baseline history rewrite file + repo_baseline_rev_tag_wcomponent_wstatus_lookup=$(grep -E "^${repo_baseline_rev_tag_wcomponent_wstatus}@.+/.+/.+$" ${execution_root_directory}/${ccm_db}_gitfiles/baseline_history_rewrite.txt | cut -d @ -f 2- ) || { echo "INFO: No rewriting baseline found - skip" ; } + if [[ ${repo_baseline_rev_tag_wcomponent_wstatus_lookup:-} != "" ]]; then + local tag_found=false + local tag_fetch_continue=true + local total_sleep_sec=0 + local sleep_sec=300 + local sleep_total_wait=86400 # 24 hours + while $tag_fetch_continue ; do + # TODO: consider to check if the baseline has changed state like pub -> rel + if git fetch ${https_remote_common} -f --no-tags refs/tags/${repo_baseline_rev_tag_wcomponent_wstatus_lookup}:refs/tags/${repo_baseline_rev_tag_wcomponent_wstatus_lookup} ; then + echo "All good - we got the tag" + [[ -d .git/lfs ]] && { + git lfs fetch ${https_remote_common} --all + git lfs fetch ${git_remote_to_use} --all + } + tag_found=true + tag_fetch_continue=false + else + if [[ ${total_sleep_sec} -gt ${sleep_total_wait} ]]; then + echo "ERROR: We waited for 24 hours - stop - exit" + exit 1 + fi + echo "Sleep: ${sleep_sec} sec of total allowed: ${sleep_total_wait} sec ( 24 hours )" + sleep $sleep_sec + total_sleep_sec=$(( total_sleep_sec + sleep_sec)) + fi + done + repo_baseline_rev_tag_wcomponent_wstatus=${repo_baseline_rev_tag_wcomponent_wstatus_lookup} + fi + fi + repo_baseline_rev_tag_wcomponent_wstatus_gitnormalized="" + byref_translate_from_ccm_version2git_tag "${repo_baseline_rev_tag_wcomponent_wstatus}" repo_baseline_rev_tag_wcomponent_wstatus_gitnormalized + echo "repo_baseline_rev_tag_wcomponent_wstatus_gitnormalized=${repo_baseline_rev_tag_wcomponent_wstatus_gitnormalized}" + + local repo_convert_rev_tag_wcomponent_wstatus="${repo_name}/${ccm_release}/${repo_convert_rev_tag}" + repo_convert_rev_tag_wcomponent_wstatus_gitnormalized="" + byref_translate_from_ccm_version2git_tag "${repo_convert_rev_tag_wcomponent_wstatus}" repo_convert_rev_tag_wcomponent_wstatus_gitnormalized + echo "repo_convert_rev_tag_wcomponent_wstatus_gitnormalized=${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" + + # Get the right content + if [ $(git describe ${repo_convert_rev_tag}) ] ; then + # we do have the correct 'content' tag checkout it out + pwd + git clean -xffd || git clean -xffd || git clean -xffd # It can happen that the first clean fails, but more tries can fix it + for path_failed_to_remove in $(git reset -q --hard ${repo_convert_rev_tag} 2>&1 | awk -F "'" '{print $2}'); do + echo "Reset/remove submodule and it's path: ${path_failed_to_remove}" + git rm -rf --cached ${path_failed_to_remove} > /dev/null 2>&1 || echo "never mind" + rm -rf ${path_failed_to_remove} + done + else + # we do not have the 'content' tag available - investigate its history if it exists ( e.g. missing in repo ) + ./ccm-baseline-history-get-root.sh "${ccm_name}${ccm_delim}${ccm_repo_convert_rev_tag}:project:${project_instance}" + exit 1 + fi + + [[ ${repo_baseline_rev_tag_wcomponent_wstatus_gitnormalized} == "" ]] && { + echo "ERROR: repo_baseline_rev_tag_wcomponent_wstatus_gitnormalized is empty - something is wrong" + exit 1 + } + # Move the workarea pointer to the 'baseline' tag + git reset -q --mixed ${repo_baseline_rev_tag_wcomponent_wstatus_gitnormalized} > /dev/null 2>&1 + if [[ "${repo_baseline_rev_tag_wcomponent_wstatus_lookup:-}" != "" ]]; then + # we dont the tag after we have reset to it - delete it again to avoid push confusion + git tag --delete "${repo_baseline_rev_tag_wcomponent_wstatus_lookup}" + fi + + # get the .gitignore files from init commit + for file in $(git ls-tree --name-only -r ${repo_name}/init/init^{}); do + git checkout ${repo_name}/init/init $file + git add ${file} + done + + exit_code=0 + if [[ ${submodules_from_baseline_obj:-} == true ]] ; then + ccm_submodules4part="$(ccm query "is_project_in_baseline_of(has_project_in_baseline('${ccm_4part}')) and not ( name='${repo_name}' )" -u -f "%objectname" )" || exit_code=$? + if [[ $exit_code -ne 0 ]]; then + if [[ $exit_code -eq 6 ]]; then + # query did not give outout - try to find the previous release tag via git describe and get setup from there + # previous _rel + echo "WARNING: No submodules found.. Restore what was already there from previous previous submodules list and content" + if git restore .gitmodules ; then + regex_submodule_line='^160000[[:blank:]]commit[[:blank:]]([0-9a-f]{40})[[:blank:]]+(.+)$' + IFS=$'\n\r' + for submodule_line in $(git ls-tree -r HEAD | grep -E '^160000[[:blank:]]+commit[[:blank:]]+[0-9a-f]{40}[[:blank:]]+.+$') ; do + echo "Process: $submodule_line" + if [[ ${submodule_line} =~ ${regex_submodule_line} ]] ; then + local submodule_sha1=${BASH_REMATCH[1]} + local submodule_path=${BASH_REMATCH[2]} + git restore ${submodule_path} + #git update-index --add --replace --cacheinfo "160000,${submodule_sha1},${submodule_path}" + unset submodule_sha1 + unset submodule_path + else + echo skip: $submodule_line +# cat .gitmodules +# exit 1 + fi + done + else + echo "INFO: no .gitmodules found" + fi + else + echo "ERROR: Something went wrong" + exit 1 + fi + fi + else + rm -f .gitmodules + if [[ ! ${repo_submodules} == "" ]]; then + touch .gitmodules + fi + ccm_submodules4part="$(ccm query "is_member_of('${ccm_4part}') and name!='${ccm_name}' and type='project'" -u -f "%objectname" )" || exit_code=$? + fi + if [[ $exit_code -ne 0 ]]; then + if [[ $exit_code -eq 6 ]]; then + # query did not give outout + repo_submodules4part="" + else + echo "ERROR: Something went wrong" + exit 1 + fi + fi + IFS=$'\n\r' + for ccm_submodule4part in ${ccm_submodules4part} ; do + set +x + [[ ${ccm_submodule4part} =~ ${regex_ccm4part} ]] || exit 1 + local ccm_submodule_name=${BASH_REMATCH[1]} + local ccm_submodule_rev=${BASH_REMATCH[2]} + local ccm_submodule_inst=${BASH_REMATCH[4]} + + local repo_submodule_name="" + byref_translate_from_ccm_name2git_repo "$ccm_submodule_name" repo_submodule_name + local repo_submodule_rev="" + byref_translate_from_ccm_version2git_tag "$ccm_submodule_rev" repo_submodule_rev + + # Lookup the subproject if present + repo_submodule=$(echo ${repo_submodules_map[${repo_submodule_name:-}]:-}) + if [[ "${repo_submodule}" == "" ]] ; then + echo "[INFO]: ${repo_submodule_name} - The subproject not found in projects to add as submodules - skip" + [[ ${debug:-} == "true" ]] && set -x + cd ${root_dir} + continue + fi + unset repo_submodule_name + echo "[INFO]: ${ccm_submodule4part} - use it" + [[ ${debug:-} == "true" ]] && set -x + + git_remote_submodule_to_use=$(echo ${git_remote_to_use} | sed -e "s/\/${repo_name}.git/\/${repo_submodule}.git/") + if [[ ${submodules_from_baseline_obj:-} == true ]] ; then + shared_config_file=$(git ls-tree -r --name-only ${repo_convert_rev_tag} | grep '^.*/shared_config.txt$') || { + if [[ $? == 1 ]] ; then + echo "[INFO]: shared_config.txt file not found - ok" + else + echo "[ERROR]: Something went wrong in finding the shared_config.txt file" + exit 1 + fi + } + if [[ "${shared_config_file:-}" != "" ]]; then + echo "[INFO]: shared_config.txt found in the git tag ${repo_convert_rev_tag}" + if ! git cat-file -p ${repo_convert_rev_tag}:$(git ls-tree -r --name-only ${repo_convert_rev_tag} | grep '^.*/shared_config.txt$' ) | grep -E "^${ccm_submodule_name}${ccm_delim}.+$|^\.\.\\\\(.+\\\\)*${ccm_submodule_name}${ccm_delim}.+$" ; then + echo "WARNING: The submodule is not to be found in shared_config_file - fall-back to default" + _path_from_shared_config_file="." + else + # The submodule reference found in shared_config file - convert to unix slashes + _path_from_shared_config_file=$(dirname $(git cat-file -p ${repo_convert_rev_tag}:$(git ls-tree -r --name-only ${repo_convert_rev_tag} | grep '^.*/shared_config.txt$') \ + | grep -E "^${ccm_submodule_name}${ccm_delim}.+$|^\.\.\\\\(.+\\\\)*${ccm_submodule_name}${ccm_delim}.+$" | sed -e 's/\\/\//g')) + fi + if [[ ${_path_from_shared_config_file} == "${ccm_submodule_name}" || ${_path_from_shared_config_file} == "." ]]; then + echo "The path was not explicitly specified in the shared_config file or module not found - add it to the dir of the shared_config.txt" + git_submodule_path="$(dirname "${shared_config_file}" | sed -e 's/\\/\//g' )/${repo_submodule}" + else + # We found a qualified path for the submodule in the shared_config file + echo "Use the found path from ${shared_config_file}: ${_path_from_shared_config_file}" + git_submodule_path="$( realpath -m --relative-to=./ $(dirname ${shared_config_file})/${_path_from_shared_config_file})/${repo_submodule}" + echo "Set path of submodule to $git_submodule_path in root folder" + fi + else + echo "Place the submodule in the root folder as fall-back" + git_submodule_path=${repo_submodule} + fi + else + git_submodule_path_in_project=$(ccm finduse -p "${ccm_submodule4part}" | grep "^[[:space:]]" | grep -v '[[:space:]]Projects:' | grep "${ccm_submodule4part//:project:1/''}" | sed -e 's/\t//g' | sed -e 's/\\/\//g' | grep -e "^.*@${ccm_4part//:project:1/''}"'$' | awk -F "${ccm_delim}" '{print $1}' | sed -e "s/^${repo_name}/./g" | sed -e "s/\/${repo_submodule}//") + git_submodule_path=${git_submodule_path_in_project}/${repo_submodule} + fi + [[ ${git_submodule_path:-} == "" ]] && ( echo "submodule path is empty - exit 1" && exit 1 ) + + case ${submodule_update_mode:-} in + "update-index") + git add -A . # just add here so execute bit can be manipulated in staged + # Get the sha1 from a reference / tag or reference is sha1 as not provided + https_remote_submodule=$(echo ${git_remote_to_use} | sed -e "s/\/me_limited_submodules.git/\/${repo_submodule}.git/") + export repo_submodule_sha1=$(git ls-remote --tag ${https_remote_submodule} | grep -e "${repo_submodule}/.*/${repo_submodule_rev}_rel$" | awk -F" " '{print $1}') + # Look then for the "pub" tag + [[ ${repo_submodule_sha1} == "" ]] && export repo_submodule_sha1=$(git ls-remote --tag ${https_remote_submodule} | grep -e "${repo_submodule}/.*/${repo_submodule_rev}_pub$" | awk -F" " '{print $1}') + # Accept what is there of remaining + [[ ${repo_submodule_sha1} == "" ]] && export repo_submodule_sha1=$(git ls-remote --tag ${https_remote_submodule} | grep -e "${repo_submodule}/.*/${repo_submodule_rev}_[dprtis][eueenq][lblsta]$" | awk -F" " '{print $1}') + + [[ ${repo_submodule_sha1} == "" ]] && exit 1 + + echo "INFO: Setting submodule: ${repo_submodule} to ${repo_submodule_sha1}" + + echo "INFO: Remove the old dir in git index" + if [[ ! $(git rm -rf ${repo_submodule} > /dev/null 2>&1 ) ]]; then + rm -rf ${repo_submodule} + fi + echo "INFO: Add the submodule to .gitmodules" + git config -f ./.gitmodules --add submodule.${repo_submodule}.path ${repo_submodule} + git config -f ./.gitmodules --add submodule.${repo_submodule}.url ../${repo_submodule}.git + git add ./.gitmodules + + echo "INFO: set the sha1: ${repo_submodule_sha1} as the reference of the submodule: ${repo_submodule}" + git update-index --add --replace --cacheinfo "160000,${repo_submodule_sha1},${repo_submodule}" + if [[ ${push_tags_in_submodules} == "true" ]]; then + #TODO: if push of tags in submodules is desired + exit 1 + https_url=$(git config --get remote.origin.url | awk -F "/" '{print $3}' | sed -e 's/git@/https:\/\//g' -e 's/7999/7990/') + curl https://api.bitbucket.org/2.0/repositories/jdoe/myrepo/refs/tags \ + -s -u jdoe -X POST -H "Content-Type: application/json" \ + -d '{ "name" : "new-tag-name", "target" : { "hash" : "a1b2c3d4e5f6" } }' + fi + git status | grep "${repo_submodule}" + cat .gitmodules + ;; + "directory") + if [[ ! $(git rm -rf ${repo_submodule}) ]]; then + rm -rf ${repo_submodule} + # This should really not be necessary # rm -rf .git/modules/${repo_submodule} + fi + if [[ ! $(git submodule update --init --recursive --force ${repo_submodule}) ]] ; then + git rm -rf ${git_submodule_path} --cached || echo "Good already - never mind" + rm -rf ${git_submodule_path} + touch .gitmodules && git add .gitmodules + if ! git submodule add --force --name "${repo_submodule}" "../${repo_submodule}.git" "${git_submodule_path}" ; then + cd ${git_submodule_path} + git remote -v + git status + pwd + git checkout -B master + git reset --hard ${repo_submodule}/${repo_init_tag}/${repo_init_tag} || { + git fetch origin +refs/tags/*:refs/tags/* + git reset --hard ${repo_submodule}/${repo_init_tag}/${repo_init_tag} + } + cd ${root_dir} + git submodule add --force --name "${repo_submodule}" "../${repo_submodule}.git" "${git_submodule_path}" + else + git submodule add --force --name "${repo_submodule}" "../${repo_submodule}.git" "${git_submodule_path}" + fi + fi + git add ./.gitmodules + + cd ${git_submodule_path} + + # Look for the "rel" tag first + git fetch ${git_remote_submodule_to_use} --tags +refs/heads/*:refs/remotes/origin/* + git_resolve_tags_wstatus "${repo_submodule}" "${repo_submodule_rev}" + if [[ "${repo_submodule_rev_wcomponent_wstatus}" == "" ]] ; then + echo "[ERROR]: Could find the revision ${repo_submodule}/.*/${repo_submodule_rev}_???" + exit 1 + fi + + if [[ ${push_tags_in_submodules} == "true" ]]; then + # root project tag handling + if [[ ! $(git describe ${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}) ]] ; then + # it was not found try and fetch to make 100% sure for whatever reason it is not here.. + git fetch ${git_remote_submodule_to_use} --tags +refs/heads/*:refs/remotes/origin/* + fi + if [[ $(git describe ${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}) ]] ; then + # we already have the correct tag, so just set it and move on.. + git reset -q --hard ${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized} + git clean -xffd + unset repo_submodule_rev + unset ccm_submodule_inst + cd ${root_dir} + continue + fi + fi + + if [ $(git describe "${repo_submodule_rev_wcomponent_wstatus}") ] ; then + # we do have the correct 'content' tag - reset hard to it and make sure we are clean.. + git clean -xffd + git reset -q --hard + git reset -q --hard "${repo_submodule_rev_wcomponent_wstatus}" + git clean -xffd + else + # we do not have the 'content' tag available - investigate its root + cd $(dirname $0) + ./ccm-baseline-history-get-root.sh "${ccm_submodule4part})" + exit 1 + fi + + if [[ ${push_tags_in_submodules} == "true" ]]; then + git tag -f -a -m "Please see tag in master repo for info: ${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" "${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" + git push ${git_remote_submodule_to_use} --recurse-submodules=no -f "${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" + fi + + cd ${root_dir} + ;; + *) + echo "[Submodule-mode] WHY are we here: submodule_update_mode is: ${submodule_update_mode} " + exit 1 + esac + + unset repo_submodule_rev + unset repo_submodule_rev_wcomponent_wstatus + unset ccm_submodule_inst + unset repo_submodule + + done + unset IFS + cd ${root_dir} + git add -A . > /dev/null 2>&1 + + if [[ ! ${repo_submodules} == "" ]]; then + if [[ -f .gitmodules ]]; then + cat .gitmodules + git add .gitmodules + else + echo "INFO: No .gitmodules initialized" + fi + fi + + git_set_execute_bit_in_index_of_extensions + git_set_execute_bit_in_index_of_unix_tool_file_executable + + export GIT_COMMITTER_DATE=$(git log -1 --format='%cd' ${repo_convert_rev_tag}) && [[ -z ${GIT_COMMITTER_DATE} ]] && return 1 + export GIT_COMMITTER_NAME=$(git log -1 --format='%cn' ${repo_convert_rev_tag} ) && [[ -z ${GIT_COMMITTER_NAME} ]] && return 1 + export GIT_COMMITTER_EMAIL=$(git log -1 --format='%ce' ${repo_convert_rev_tag} ) && [[ -z ${GIT_COMMITTER_EMAIL} ]] && return 1 + + export GIT_AUTHOR_DATE=$(git log -1 --format='%ad' ${repo_convert_rev_tag} ) && [[ -z ${GIT_AUTHOR_DATE} ]] && return 1 + export GIT_AUTHOR_NAME=$(git log -1 --format='%an' ${repo_convert_rev_tag} ) && [[ -z ${GIT_AUTHOR_NAME} ]] && return 1 + export GIT_AUTHOR_EMAIL=$(git log -1 --format='%ae' ${repo_convert_rev_tag} ) && [[ -z ${GIT_AUTHOR_EMAIL} ]] && return 1 + + echo "git commit content of ${repo_convert_rev_tag}" + git commit -C ${repo_convert_rev_tag} --reset-author || ( echo "Empty commit.." ) + + git submodule status || { + exit_code=$? + cat .gitmodules + exit $exit_code + } + git status + + set +x + echo "#####################" + echo "# git sizes:" + echo "#####################" + [[ -d .git/lfs ]] && printf "INFO: size: %s %s\n" $(du -sh .git/lfs) + [[ -d .git/lfs ]] && printf "INFO: LFS files count in this commit: %s\n" $( git lfs ls-files HEAD | wc -l ) + [[ -d .git/objects ]] && printf "INFO: size: %s %s\n" $(du -sh .git/objects) + [[ -d .git/modules ]] && printf "INFO: size: %s %s\n" $(du -sh .git/modules) + echo "#####################" + [[ ${debug:-} == "true" ]] && set -x + + # reset the committer to get the correct set for the commiting the tag. There is no author of the tag + export GIT_AUTHOR_DATE=$(git tag -l --format="%(taggerdate:iso8601)" "${repo_convert_rev_tag}" | awk -F" " '{print $1 " " $2}') && [[ -z ${GIT_AUTHOR_DATE} ]] && return 1 + export GIT_COMMITTER_DATE=${GIT_AUTHOR_DATE} + export GIT_COMMITTER_NAME=$(git tag -l --format="%(taggername)" "${repo_convert_rev_tag}" ) && [[ -z ${GIT_COMMITTER_NAME} ]] && return 1 + export GIT_COMMITTER_EMAIL=$(git tag -l --format="%(taggeremail)" "${repo_convert_rev_tag}" ) && [[ -z ${GIT_COMMITTER_EMAIL} ]] && return 1 + + echo "Get tag content of: ${repo_convert_rev_tag}" + git tag -l --format '%(contents)' "${repo_convert_rev_tag}" > ./tag_meta_data.txt + echo "git commit content of ${repo_convert_rev_tag}" + echo "git tag ${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized} based on ${repo_convert_rev_tag}" + git tag -a -F ./tag_meta_data.txt "${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" + rm -f ./tag_meta_data.txt + + # Do not consider submodules + if [[ ${push_to_remote_during_conversion:-} == "true" ]]; then + echo "INFO: Configured to push to remote: git push ${git_remote_to_use} --recurse-submodules=no -f ${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" + git push ${git_remote_to_use} --recurse-submodules=no -f "${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" || { + if [[ ${https_remote_common:-} != "" ]]; then + [[ -d .git/lfs ]] && { + git lfs fetch ${https_remote_common} --all + git lfs fetch ${git_remote_to_use} --all + } + git push ${git_remote_to_use} --recurse-submodules=no -f "${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" + else + echo "ERROR: something when wrong in pushing... and it is not due to use of common repo" + fi + } + if [[ ${https_remote_common:-} != "" ]]; then + git push ${https_remote_common} --recurse-submodules=no -f "${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" || { + [[ -d .git/lfs ]] && { + git lfs fetch ${https_remote_common} --all + git lfs fetch ${git_remote_to_use} --all + git push ${git_remote_submodule_to_use} --recurse-submodules=no -f "${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" + } + } + fi + else + echo "INFO: Skip push to remote" + fi + + set +x + echo "============================================================================" + echo " DONE: $repo_convert_rev_tag_wcomponent_wstatus" + echo "============================================================================" + [[ ${debug:-} == "true" ]] && set -x + + unset GIT_AUTHOR_DATE + unset GIT_AUTHOR_NAME + unset GIT_AUTHOR_EMAIL + unset GIT_COMMITTER_DATE + unset GIT_COMMITTER_NAME + unset GIT_COMMITTER_EMAIL + unset root_dir + unset tag_to_convert + unset repo_convert_rev_tag + unset repo_convert_rev_tag_wcomponent_wstatus + unset repo_convert_rev_tag_wcomponent_wstatus_gitnormalized + unset repo_baseline_rev_tag_wcomponent_wstatus + unset repo_baseline_rev_tag_wcomponent_wstatus_lookup + unset repo_baseline_rev_tag_wcomponent_wstatus_gitnormalized + unset ccm_repo_convert_rev_tag + # From subfunction + unset proj_name + unset proj_version + unset proj_instance + unset project_release + unset ccm_baseline_obj_and_status_release_this + unset ccm_baseline_obj + unset ccm_baseline_status + unset ccm_baseline_release + +} + +function reset_converted_tags_except_init_remote_n_local() { + echo "Delete all local and remote tags '^${repo_name}/.*/.*_[dprtis][eueenq][lblsta]$'" + git tag | grep "^${repo_name}/.*/.*_[dprtis][eueenq][lblsta]$" | xargs --no-run-if-empty git push ${git_remote_to_use} --delete || echo "Some tags might not be on the remote - never mind" + git tag | grep "^${repo_name}/.*/.*_[dprtis][eueenq][lblsta]$" | xargs --no-run-if-empty git tag --delete || echo "Some tags might not be on the remote - never mind" +} + +function reset_converted_init_tag_remote_n_local() { + echo "Delete local and remote tag ${repo_name}/${repo_init_tag}/${repo_init_tag}" + git tag | grep "^${repo_name}/${repo_init_tag}/${repo_init_tag}$" | xargs --no-run-if-empty git push ${git_remote_to_use} --delete || echo "Some tags might not be on the remote - never mind" + git tag | grep "^${repo_name}/${repo_init_tag}/${repo_init_tag}$" | xargs --no-run-if-empty git tag --delete || echo "Some tags might not be on the remote - never mind" +} + +lock_repo_init_file="${execution_root_directory}/repo_under_construction_lock.txt" + +if [[ -f ${lock_repo_init_file} ]]; then + echo "INFO: Init construction of repo $repo_name did not complete - restart process" + rm -rf $repo_name +fi + +if [[ "${execute_mode}" == "reclone" ]]; then + echo "INFO: execute_mode is: '${execute_mode}'" + rm -rf ${repo_name} +fi + +if [[ ! -d "${repo_name}" ]] ; then + #initialize repo + echo "LOCK Repo: ${repo_name} cloned from: ${git_remote_to_use} is under init construction" > ${lock_repo_init_file} + git clone ${git_remote_to_use} + cd ${repo_name} + git fetch ${git_remote_to_use_orig} --tags --force +refs/heads/*:refs/remotes/origin/* + git branch -a + git tag + git reset -q --hard ${repo_init_tag} + git clean -xffd + + reset_converted_tags_except_init_remote_n_local + reset_converted_init_tag_remote_n_local + + export GIT_AUTHOR_DATE=$(git tag -l --format="%(taggerdate:iso8601)" ${repo_init_tag} | awk -F" " '{print $1 " " $2}') + export GIT_COMMITTER_DATE=${GIT_AUTHOR_DATE} + + for gitignore_path_n_file in $(echo ${gitignore_path_n_files} | sed -e 's/:/ /g'); do + gitignore_file_name=$(echo ${gitignore_path_n_file} | cut -d "@" -f 1) + gitignore_rel_path=$(echo ${gitignore_path_n_file} | cut -d "@" -f 2) + gitignore_full_path_name="${execution_root_directory}/${gitignore_file_name}" + if [[ ! -f ${gitignore_full_path_name} ]]; then + echo "${gitignore_full_path_name} does not exist.. Current dir:" + pwd + echo " .. Consider full path.." + exit 1 + else + mkdir -p ${gitignore_rel_path} + cp ${gitignore_full_path_name} ${gitignore_rel_path}/.gitignore + fi + done + + git_initialize_lfs_n_settings + for gitattributes_path_n_file in $(echo ${gitattributes_path_n_files} | sed -e 's/:/ /g'); do + gitattributes_file_name=$(echo ${gitattributes_path_n_file} | cut -d "@" -f 1) + gitattributes_rel_path=$(echo ${gitattributes_path_n_file} | cut -d "@" -f 2) + gitattributes_full_path_name="${execution_root_directory}/${gitattributes_file_name}" + if [[ ! -f ${gitattributes_full_path_name} ]]; then + echo "${gitattributes_full_path_name} does not exist.. - skip" + else + mkdir -p ${gitattributes_rel_path} + cp ${gitattributes_full_path_name} ${gitattributes_rel_path}/.gitattributes + fi + done + + git add -A . + git status + + echo "git commit init : ${repo_name}/${repo_init_tag}/${repo_init_tag}" + git commit -C "$repo_init_tag" --amend --reset-author + echo "git tag init commit: ${repo_name}/${repo_init_tag}/${repo_init_tag}" + git tag -f -a -m $(git tag -l --format '%(contents)' ${repo_init_tag}) ${repo_name}/${repo_init_tag}/${repo_init_tag} + + unset GIT_AUTHOR_DATE + unset GIT_COMMITTER_DATE + + git reset -q --hard ${repo_name}/${repo_init_tag}/${repo_init_tag} + git clean -xffd + + git ls-tree -r ${repo_name}/${repo_init_tag}/${repo_init_tag} + + if [[ ${push_to_remote_during_conversion:-} == "true" ]]; then + echo "INFO: Configured to push to remote: git push ${git_remote_to_use} --recurse-submodules=no -f ${repo_name}/${repo_init_tag}/${repo_init_tag}" + git push ${git_remote_to_use} --recurse-submodules=no -f ${repo_name}/${repo_init_tag}/${repo_init_tag} + else + echo "INFO: Skip push to remote: ${git_remote_to_use}" + fi + + pwd # we are still in the root repo + echo "UNLOCK repo: ${repo_name} as init construction completed.." && rm -f ${lock_repo_init_file} +else + echo "Already cloned and initialized" + cd ${repo_name} + if [[ "${execute_mode}" == "normal" ]]; then + echo "INFO: execute_mode is: '${execute_mode}'" + echo "Reset local tags in scope '^${repo_name}/.*/.*_[dprtis][eueenq][lblsta]$' and then start from begin of '^${repo_name}/init/init$'" + if [[ ! $( git tag | grep -v "^${repo_name}/init/init$" | grep "^${repo_name}/.*/.*_[dprtis][eueenq][lblsta]$" | xargs --no-run-if-empty git tag --delete ) ]] ; then + echo "No tags found" + fi + git fetch ${git_remote_to_use} --tags --force +refs/heads/*:refs/remotes/origin/* + git fetch ${git_remote_to_use} -ap +refs/heads/*:refs/remotes/origin/* + elif [[ "${execute_mode}" == "continue_locally" ]];then + echo "INFO: execute_mode is: '${execute_mode}'" + echo "Do not delete already converted tags and fetch again - just continue in workspace as is" + elif [[ "${execute_mode}" == "reset_remote_n_local" ]];then + echo "INFO: execute_mode is: '${execute_mode}'" + git fetch ${git_remote_to_use} --tags --force +refs/heads/*:refs/remotes/origin/* + git fetch ${git_remote_to_use} -ap +refs/heads/*:refs/remotes/origin/* + reset_converted_tags_except_init_remote_n_local + fi + git_initialize_lfs_n_settings +fi + +for sha1 in $(git log --topo-order --oneline --all --pretty=format:"%H " | tac) ; do + echo "Processing: $sha1" + tags=$(git tag --points-at "${sha1}" | grep -v "^${repo_name}/init/init$" | grep -v "^.*/.*/.*_[dprtis][eueenq][lblsta]$" | grep -v '/' | grep "^.*_[dprtis][eueenq][lblsta]$" || echo "") + if [[ "${tags}" == "" ]]; then + converted_tags=$(git tag --points-at "${sha1}" | grep .*/.*/.*_[dprtis][eueenq][lblsta]$ || echo "") + echo "INFO : No unconverted tags found - These are the new tags found - list and continue" + echo "${converted_tags}" + continue + fi + for project_revision in $(git tag --points-at "${sha1}" | grep -v "^${repo_name}/init/init$" | grep -v "^.*/.*/.*_[dprtis][eueenq][lblsta]$" | grep "^.*_[dprtis][eueenq][lblsta]$" || echo "@@@" ); do + [[ "${repo_name}/${repo_init_tag}/${repo_init_tag}" == "${project_revision}" ]] && continue + [[ "${repo_init_tag}" == "${project_revision}" ]] && continue + [[ "@@@" == "${project_revision}" ]] && continue + convert_revision ${project_revision} + done + echo "Done: $sha1" +done + +[[ -d .git/lfs ]] && printf "Git LFS in total: %s\n" $( git lfs ls-files --all | wc -l ) +[[ -d .git/lfs ]] && echo "Store list of LFS files in: ${execution_root_directory}/git_lfs_files.txt" +[[ -d .git/lfs ]] && git lfs ls-files --all > ${execution_root_directory}/git_lfs_files.txt + + diff --git a/ccm-extract-baseline-project-metadata.sh b/ccm-extract-baseline-project-metadata.sh new file mode 100644 index 0000000..620cf65 --- /dev/null +++ b/ccm-extract-baseline-project-metadata.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash +set -e +set -u + +[[ "${debug:-}" == "true" ]] && set -x + +# Load functions +source $(dirname $0)/_ccm-functions.sh || source ./_ccm-functions.sh + +[[ -z $1 ]] && ( echo "Please set parameter 1 to ccm 4 part project name - exit 1" >&2 && exit 1 ) +ccm_4name=$1 + +[[ "${ccm_4name}" =~ ${regex_ccm4part} ]] || { + echo "$0: 4part does not comply" + exit 1 + } +ccm_project_name=${BASH_REMATCH[1]} +repo_convert_rev_tag=${BASH_REMATCH[2]} +repo_convert_type=${BASH_REMATCH[3]} +repo_convert_instance=${BASH_REMATCH[4]} + + +[[ -z ${2:-} ]] && ( echo "Please set parameter 2 to Jira Project Key - exit 1" >&2 && exit 1 ) +jira_project_key=$2 + +[[ -z ${3:-} ]] && ( echo "Please set parameter 3 to 'commit' or 'tag' - exit 1" >&2 && exit 1 ) +target_type=$3 + +jira_task_to_jira_issue_base=9000000 + +if [[ $target_type == "tag" ]] ; then + extract_data_epic_level="false" + extract_data_story_level="true" + extract_data_ccm_task_level="false" + extract_data_ccm_task_handle_dirtytasks_separately="false" + extract_data_ccm_task_verbosed_level="false" +elif [[ $target_type == "commit" ]] ; then + extract_data_epic_level="false" + extract_data_story_level="true" + extract_data_ccm_task_level="true" + extract_data_ccm_task_handle_dirtytasks_separately="false" + extract_data_ccm_task_verbosed_level="false" +else + echo "Parameter 5 is not set to 'commit' or 'tag' - exit 1" >&2 + exit 1 +fi + +# FIXME: This is a bit of a hack to determine the database type - should be done as a parameter/config as the database config +ccm_current_db=$(ccm status -f "%database %current_session" | grep TRUE | awk -F " " '{print $1}') +case ${ccm_current_db} in + /data/ccmdb/db_x|/data/ccmdb/db_y) + epic_level_header="Change Requests: (CR)" + epic_level_release_attr="TargetRelease" + epic_level_epic2story_relation="associatedWP" + + story_level_header="Work Packages (WP)" + story_level_release_attr="TargetRelease" + require_baseline_object="false" + ;; + /data/ccmdb/z) + epic_level_header="Master Change Requests: (MCR)" + epic_level_release_attr="release" + epic_level_epic2story_relation="associatedImpl" + + story_level_header="Implementation Change Requests(ICR)" + story_level_release_attr="release" + require_baseline_object="false" + ;; + *) + epic_level_header="Master Change Requests: (MCR)" + epic_level_release_attr="release" + epic_level_epic2story_relation="associatedImpl" + story_level_header="Implementation Change Requests" + story_level_release_attr="release" + require_baseline_object="false" + ;; +esac + +test "${ccm_project_name}x" == "x" && ( echo "'ccm_project_name' not set - exit" >&2 && exit 1 ) +test "${repo_convert_rev_tag}x" == "x" && ( echo "'repo_convert_rev_tag' not set - exit" >&2 && exit 1 ) +test "${repo_convert_instance}x" == "x" && ( echo "'repo_convert_rev_tag' not set - exit" >&2 && exit 1 ) + +output_file="./meta_data.txt" +rm -f ${output_file} + +function handle_task_attrs { + local _task_number_attrs=$1 + if [[ ${_task_number_attrs} == "none" ]] ; then + echo "">> ${output_file} + else + # 1 2 3 4 5 6 + regex='^(.+)@@@(.+)@@@(.+)@@@(.+)@@@(.+)@@@(.+)$' + [[ ${_task_number_attrs} =~ $regex ]] || exit 1 + task_objectname=${BASH_REMATCH[1]} + task_number=${BASH_REMATCH[2]} + task_create_time=${BASH_REMATCH[3]} + task_resolver=${BASH_REMATCH[4]} + task_status=${BASH_REMATCH[5]} + task_release="${BASH_REMATCH[6]}" + task_synopsis=$(ccm attr -show task_synopsis ${task_objectname}) + jira_subtask_issue_number=$(($jira_task_to_jira_issue_base + $task_number)) + printf "%-5s %-15s %-10s %-6s %-9s %-30s %s\n" " $loop_number)" "${jira_project_key}-${jira_subtask_issue_number}" "$task_create_time" "$task_resolver" "$task_status" "$task_release" "$task_synopsis" >> ${output_file} + fi +} +exit_code="0" +find_n_set_baseline_obj_attrs_from_project "${ccm_project_name}${ccm_delim}${repo_convert_rev_tag}:project:${repo_convert_instance}" "verbose_false" || exit_code=$? +if [[ "${exit_code}" != "0" ]] ; then + echo "ERROR: Project not found: ${ccm_project_name}${ccm_delim}${repo_convert_rev_tag}:project:${repo_convert_instance}" >&2 + exit ${exit_code} +fi + +if [[ "${ccm_baseline_obj:-}" != "" ]]; then + objectname="${ccm_project_name}${ccm_delim}${repo_convert_rev_tag}:project:${repo_convert_instance}" + printf "Project: ${objectname} <-> Baseline object: ${ccm_baseline_obj}\n\n" >> ${output_file} + ccm baseline -show info "${ccm_baseline_obj}" -f " Build: %build\n Description: %description\n Release: %release\n Purpose: %purpose\n" >> ${output_file} + + ccm baseline -show projects "${ccm_baseline_obj}" -f "%objectname %release %owner %{create_time[dateformat='yyyy-MM-dd HH:MM:SS']} Baseline: -> %baseline" >> ${output_file} + + printf "\nAll baseline objects related to project: ${ccm_project_name}${ccm_delim}${repo_convert_rev_tag}:project:${repo_convert_instance}\n" >> ${output_file} + ccm query "has_project_in_baseline('${objectname}')" \ + -sby create_time -f "%objectname %release %create_time" >> ${output_file} + echo >> ${output_file} + + if [[ $extract_data_epic_level == "true" ]]; then + echo >> ${output_file} + echo "${epic_level_header}:" >> ${output_file} + ccm query "has_${epic_level_epic2story_relation}(has_associated_task((is_task_in_baseline_of('${ccm_baseline_obj}') or is_dirty_task_in_baseline_of('${ccm_baseline_obj}'))))" -f "${jira_project_key}-%problem_number %resolver %${epic_level_release_attr} %problem_synopsis" >> ${output_file} || echo "" >> ${output_file} + fi + + if [[ $extract_data_story_level == "true" ]]; then + echo >> ${output_file} + echo "Fully integrated ${story_level_header}:" >> ${output_file} + ccm baseline -show fully_included_change_requests -groupby "${epic_level_release_attr}: %${epic_level_release_attr}" -f "${jira_project_key}-%problem_number %resolver %${epic_level_release_attr} %problem_synopsis" "${ccm_baseline_obj}" >> ${output_file} || echo "" >> ${output_file} + + echo >> ${output_file} + echo "Partially integrated ${story_level_header}:" >> ${output_file} + ccm baseline -show partially_included_change_requests -groupby "${epic_level_release_attr}: %${epic_level_release_attr}" -f "${jira_project_key}-%problem_number %resolver %${epic_level_release_attr} %problem_synopsis" "${ccm_baseline_obj}" >> ${output_file} || echo "" >> ${output_file} + fi + + if [[ $extract_data_ccm_task_level == "true" ]]; then + echo >> ${output_file} + if [[ ${extract_data_ccm_task_handle_dirtytasks_separately} == "true" ]]; then + echo "Tasks integrated in baseline:" >> ${output_file} + query1="is_task_in_baseline_of('${ccm_baseline_obj}')" + query2="is_dirtytask_in_baseline_of('${ccm_baseline_obj}')" + else + echo "All tasks integrated in baseline:" >> ${output_file} + query1="is_task_in_baseline_of('${ccm_baseline_obj}') or is_dirty_task_in_baseline_of('${ccm_baseline_obj}')" + fi + + IFS=$'\n\r' + loop_number=1 + for task_number_attrs in $(ccm query "${query1}" -u -f "%objectname@@@%task_number@@@%{create_time[dateformat='yyyy-MM-dd HH:MM:SS']}@@@%resolver@@@%status@@@%release" || ( [[ $? == 6 ]] && echo "none" ) ) ; do + handle_task_attrs "$task_number_attrs" + loop_number=$((loop_number + 1)) + done + unset IFS + + if [[ ${extract_data_ccm_task_handle_dirtytasks_separately} == "true" ]]; then + echo >> ${output_file} + echo "Dirty tasks integrated in baseline: ( listed in baseline, but has not effect as it's objects are behind the baseline project )" >> ${output_file} + IFS=$'\n\r' + loop_number=1 + for task_number_attrs in $(ccm query "${query2}" -u -f "%objectname@@@%task_number@@@%{create_time[dateformat='yyyy-MM-dd HH:MM:SS']}@@@%resolver@@@%status@@@%release" || ( [[ $? == 6 ]] && echo "none" ) ) ; do + handle_task_attrs "$task_number_attrs" + loop_number=$((loop_number + 1)) + done + unset IFS + fi + if [[ $extract_data_ccm_task_verbosed_level == "true" ]]; then + echo >> ${output_file} + echo "Tasks integrated in baseline and/or project (verbosed):" >> ${output_file} + if [[ ${query2} == "" ]]; then + query3="${query1}" + else + query3="${query1} or ${query2}" + fi + + integrated_tasks=$(ccm query "${query3}" || ( [[ $? == 6 ]] && echo "none" )) + if [[ ${integrated_tasks} == "none" ]]; then + echo "" >> ${output_file} + else + ccm task -sh info -v @ >> ${output_file} + fi + echo >> ${output_file} + fi + fi + + +else + [[ "${require_baseline_object}" == "true" ]] && ( echo "ERROR: It is expected to have a baseline object due to configuration: require_baseline_object=true for this database: ${ccm_current_db}" >&2 && exit 2 ) + objectname="${ccm_project_name}${ccm_delim}${repo_convert_rev_tag}:project:${repo_convert_instance}" + + printf "Project: ${objectname} <-> Baseline object: NONE\n\n" >> ${output_file} + + echo "Project baseline:" >> ${output_file} + ccm query "is_baseline_project_of('${objectname}')" -f "%displayname" >> ${output_file} || echo " " >> ${output_file} + echo >> ${output_file} + + if [[ ${epic_level_header:-} != "" ]]; then + echo "${epic_level_header}:" >> ${output_file} + ccm query "has_${epic_level_epic2story_relation}(has_associated_task(is_task_in_folder_of(is_folder_in_rp_of('${objectname}'))))" -f "${jira_project_key}-%problem_number %resolver %release %problem_synopsis" >> ${output_file} || echo "" >> ${output_file} + + echo >> ${output_file} + fi + + echo "Related/Integrated ${story_level_header}:" >> ${output_file} + ccm query "has_associated_task(is_task_in_folder_of(is_folder_in_rp_of('${objectname}')))" -f "${jira_project_key}-%problem_number%resolver %release %problem_synopsis" >> ${output_file} || echo "" >> ${output_file} + + echo >> ${output_file} + echo "Tasks integrated in project:" >> ${output_file} + #ccm query "is_task_in_folder_of(is_folder_in_rp_of('${objectname}'))" -f "%displayname %{create_time[dateformat='yyyy-M-dd HH:MM:SS']} %resolver %status %release %task_synopsis" >> ${output_file} || echo "" >> ${output_file} + IFS=$'\n\r' + loop_number=1 + for task_number_attrs in $(ccm query "status!='task_automatic' and (is_task_in_folder_of(is_folder_in_rp_of('${objectname}')) or is_task_in_rp_of('${objectname}'))" -u -f "%objectname@@@%task_number@@@%{create_time[dateformat='yyyy-MM-dd HH:MM:SS']}@@@%resolver@@@%status@@@%release" | tail -n +2) ; do + handle_task_attrs "$task_number_attrs" + loop_number=$((loop_number + 1)) + done + unset IFS + + if [[ ${extract_data_ccm_task_verbosed_level:-} == "true" ]]; then + echo >> ${output_file} + echo "Tasks integrated in baseline and/or project (verbosed):" >> ${output_file} + integrated_tasks=$(ccm query "status!='task_automatic' and (is_task_in_folder_of(is_folder_in_rp_of('${objectname}')) or is_task_in_rp_of('${objectname}'))" || ( [[ $? == 6 ]] && echo "none" )) + if [[ ${integrated_tasks} == "none" ]]; then + echo "" >> ${output_file} + else + ccm task -sh info -v @ >> ${output_file} + fi + echo >> ${output_file} + fi + +fi +cat ${output_file} +rm -f ${output_file} \ No newline at end of file diff --git a/ccm-get-status-from-baseline-or-project.sh b/ccm-get-status-from-baseline-or-project.sh new file mode 100644 index 0000000..aad1d38 --- /dev/null +++ b/ccm-get-status-from-baseline-or-project.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -u +set -e + +[[ "${debug:-}" == "true" ]] && set -x + +# Load functions +source $(dirname $0)/_ccm-functions.sh || source ./_ccm-functions.sh + +ccm_proj_obj_string="$1" + +exit_code="0" +find_n_set_baseline_obj_attrs_from_project "${ccm_proj_obj_string}" "verbose_false" || exit_code=$? +if [[ "${exit_code}" != "0" ]] ; then + echo "ERROR: Project not found: ${ccm_proj_obj_string}" + exit ${exit_code} +fi + +if [[ "${ccm_baseline_status:-}" == "" ]]; then + # We could not set status from baseline object - take it from the project + ccm_baseline_status=$(ccm attr -show status "${ccm_proj_obj_string}" | sed -e 's/ //g' | cut -c1-3) +else + ccm_baseline_status=$(echo ${ccm_baseline_status} | cut -c1-3) +fi +if [[ "${ccm_baseline_status:-}" == "" ]] ; then + echo "Something went wrong as no status is set" + exit 1 +else + echo ${ccm_baseline_status} +fi \ No newline at end of file diff --git a/ccm-get-translated_string.sh b/ccm-get-translated_string.sh new file mode 100644 index 0000000..e7aacc5 --- /dev/null +++ b/ccm-get-translated_string.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -u +set -e + +[[ "${debug:-}" == "true" ]] && set -x + +# Load functions +source $(dirname $0)/_ccm-functions.sh || source ./_ccm-functions.sh + +byref_function=$1 +shift +[[ "${byref_function:-}" == "" ]] && ( echo "'byref_function' not set as parameter 1 - exit" && exit 1 ) + +result="" +case $byref_function in + byref_translate_from_ccm_name2git_repo) + ccm_project_name=$1 + eval $byref_function "\"${ccm_project_name}\"" result + ;; + *) + eval $byref_function $@ result + ;; +esac + +printf "$result" diff --git a/ccm.user.properties b/ccm.user.properties new file mode 100644 index 0000000..ebfb41d --- /dev/null +++ b/ccm.user.properties @@ -0,0 +1 @@ +ccm.cli.format.date=yyyy-MM-dd HH\:MM\:SS \ No newline at end of file diff --git a/ccm_find_dangeling_tasks.sh b/ccm_find_dangeling_tasks.sh new file mode 100644 index 0000000..bcc5539 --- /dev/null +++ b/ccm_find_dangeling_tasks.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash + +set -euo pipefail + +[[ ${debug:-} == true ]] && set -x + + +#Load DB settings +source ${BASH_SOURCE%/*}/${ccm_db}_settings.sh 2> /dev/null || source ./${ccm_db}_settings.sh + +if [[ ${type:-} == "task_assigned" || ${1:-} == "task_assigned" ]]; then + query="type='task' and release match '*' and status='task_assigned'" + time_date_field="create_time" + subdir=task_assigned + [[ ${jira_issue_state:-} == "" ]] && jira_issue_state="TO DO" + echo "Running in task_assigned mode" +else + query="type='task' and release match '*' and status='completed' and ( is_no_task_in_baseline() ) " + time_date_field="completion_date" + subdir=completed + [[ ${jira_issue_state:-} == "" ]] && jira_issue_state="IN VALIDATION" + echo "Running in task completed mode" +fi + +[[ ${objects_max:-} == "" ]] && objects_max=50 + +echo $query +ccm query "$query" -u -f %objectname | wc -l + +mkdir -p ${subdir} +rm -rf ${subdir}/dirty.txt + +function investigate_projects { + for project in $( ccm finduse ${task_object} -all_projs | sed -e 's/\t\t//' ); do + project_status=$(ccm attr -show status $project) + if [[ $project_status == "integrate" || $project_status == "released" || project_status == 'sqa' || project_status == "test" ]]; then + project_found=true + echo "ERROR : detect conflicts for the projects due to 'folder_in_rp' can be updated after checkin" + exit 1 + printf "OK: $task_number $project $project_status\n" >> ${subdir}/overview.txt + printf "OK: %s : %s : %s : %s : %s : %s\n" \ + "$task_number" \ + "$(ccm attr -show release ${task_object})" \ + "$(ccm attr -show status ${task_object})" \ + "$(ccm attr -show resolver ${task_object})" \ + "$(ccm attr -show ${time_date_field} ${task_object})" \ + "$(ccm attr -show task_synopsis ${task_object})" >> ${subdir}/overview.txt + + ccm finduse ${task_object} -all_projs >> ${subdir}/in_static_project/$task + return + fi + done +} + +#yyyy-MM-dd HH:mm:ss + +IFS=$'\r\n' +for task_number_resolver in $(ccm query "$query" -u -f "%task_number@%resolver" ) ; do + task_number=${task_number_resolver%@*} + task_resolver=${task_number_resolver#*@} + temp_dir_path=${subdir}/_${task_number}-${task_resolver} + final_dir_path=${subdir}/${task_number}-${task_resolver} + + if [[ -d ${final_dir_path} ]]; then + printf "|" + continue + else + printf '\n' + fi + rm -rf ${temp_dir_path} + rm -f ${temp_dir_path}/*.commit ${temp_dir_path}/*.csv + + task_object=$(ccm query "task('$task_number')" -u -f %objectname) + + + baseline_found=false + # sleep 1 + printf "${task_number}: " + for baseline in $(ccm query "has_dirty_task_in_baseline('${task_object}') or has_task_in_baseline('${task_object}') " || true ) ; do + baseline_found=true + [[ ${details:-} == true ]] && echo $baseline + done + + if [[ ${baseline_found:-} == false ]] ; then + project_found=false + #investigate_projects + if [[ ${project_found} == false ]]; then + object_count="$(ccm query "is_associated_cv_of('$task_object') and not (type='dir' or type='project')" -u -f "%objectname" | wc -l)" || exit_code=$? + if [[ $object_count -eq 0 ]]; then + printf " $object_count - skip" + continue + fi + mkdir -p ${temp_dir_path} + printf " $object_count " + printf "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n" \ + "\"${task_number}\"" \ + "\"${jira_issue_type}\"" \ + "\"${jira_issue_state}\"" \ + "\"${jira_issue_epic_state:-New}\"" \ + "\"$(ccm attr -show release ${task_object})\"" \ + "\"${task_resolver}\"" \ + "\"${task_resolver}\"" \ + "\"$(ccm attr -show ${time_date_field} ${task_object})\"" \ + "\"$(ccm attr -show ${time_date_field} ${task_object})\"" \ + "\"${object_count}\"" \ + "\"${task_number}-dangTask [ccmTask]: $(ccm attr -show task_synopsis ${task_object})\"" \ + "\"${task_number}-dangTask [ccmTask]: $(ccm attr -show task_synopsis ${task_object})\"" \ + "\"Details: ${details_link}/${final_dir_path}\"" \ + "\"${subdir}\"" > ${temp_dir_path}/task_info_${task_number}.csv + printf "%s-%s : %s\n" \ + "${jira_project_key}" \ + "${task_number}" \ + "${task_number}-dangTask [ccmTask]: $(ccm attr -show task_synopsis ${task_object})" \ + > ${temp_dir_path}/${task_number}.commit + ccm task -show info -v $task_number > ${temp_dir_path}/task_info_${task_number}.txt + if [[ ${folders:-} == true ]]; then + folders_found=false + { + printf "\n\nFolders:\n" + for folder in $( ccm finduse ${task_object} -all_folders | grep -A 100 Folder | sed -e 's/\t\t//' ); do + folders_found=true + done + echo + } >> ${temp_dir_path}/task_info_${task_number}.txt + fi + if [[ ${objects:-} == true ]]; then + echo >> ${temp_dir_path}/task_info_${task_number}.txt + printf "\n\nObjects and predecessors:\n" >> ${temp_dir_path}/task_info_${task_number}.txt + + if [[ $object_count -gt ${objects_max} ]]; then + printf "over 50 objects - skipped" + echo " - over 50 objects: $object_count - skipped" >> ${temp_dir_path}/task_info_${task_number}.txt + else + for object in $(ccm query "is_associated_cv_of('$task_object') and not (type='dir' or type='project')" -u -f "%objectname" ); do + echo "+ $object" >> ${temp_dir_path}/task_info_${task_number}.txt + ccm finduse -all_projs $object >> ${temp_dir_path}/task_info_${task_number}.txt + ccm query "has_successor('${object}')" -u -f " - predecessor: %objectname" >> ${temp_dir_path}/task_info_${task_number}.txt || { + exit_code=$? + [[ $exit_code != 6 ]] && { echo "ERROR: something is not right in predecessor of $task_number - $object"; exit $exit_code ; } + } + if [[ -e ${temp_dir_path}/${object} ]]; then + printf "." + else + printf "," + ccm cat $object > ${temp_dir_path}/_${object} + mv ${temp_dir_path}/_${object} ${temp_dir_path}/${object} + fi + done + fi + echo >> ${temp_dir_path}/task_info_${task_number}.txt + fi + mv ${temp_dir_path} ${final_dir_path} + fi + else + printf "WARNING: likely dirty" + printf "%s - skip - likely dirty\n" "$task_number" >> ${subdir}/dirty.txt + fi +done +echo + +echo "Collection csv for Jira" +echo "Issue id ,Issue Type ,Status, Epic Status, FixVersion,Assignee,Reporter,Date created,Date modified,CCM Objects, Summary,Epic Name,Description,Labels" > ${subdir}/all.csv + +find ${subdir} -name task_info_*.csv | xargs -I % cat % >> ${subdir}/all.csv +ls -la ${subdir}/all.csv + +echo "Collecting commit messages" +printf "${subdir}: Task information and objects\n\n" > ${subdir}/all.commit +find ${subdir} -name [[:digit:]]*.commit | xargs -I % cat % >> ${subdir}/all.commit +ls -la ${subdir}/all.commit +echo \ No newline at end of file diff --git a/docker.sh b/docker.sh new file mode 100755 index 0000000..c0ad786 --- /dev/null +++ b/docker.sh @@ -0,0 +1 @@ +docker build -t 2git dockerfiles/jdk17/Dockerfile . diff --git a/dockerfiles/jdk17/Dockerfile b/dockerfiles/jdk17/Dockerfile new file mode 100644 index 0000000..cb20f1e --- /dev/null +++ b/dockerfiles/jdk17/Dockerfile @@ -0,0 +1,10 @@ +FROM gradle:jdk17-alpine as build +WORKDIR /src +COPY . /src/ +RUN gradle uberjar + +FROM openjdk:17-alpine +WORKDIR /app +COPY --from=0 /src/build/libs/*.jar /app/2git.jar + +CMD ["java","-jar", "/app/2git.jar"] \ No newline at end of file diff --git a/emptydir.gitignore b/emptydir.gitignore new file mode 100644 index 0000000..407062b --- /dev/null +++ b/emptydir.gitignore @@ -0,0 +1,4 @@ +# Ignore everything - added in CM/synergy to git migration as it was an empty directory +* +# except .gitignore +!.gitignore diff --git a/execute-ccm-ccm2git.sh b/execute-ccm-ccm2git.sh new file mode 100644 index 0000000..4dbd7d0 --- /dev/null +++ b/execute-ccm-ccm2git.sh @@ -0,0 +1,206 @@ +#!/bin/bash --login +set -e +set -u + +# Load functions + +[[ ${ccm_db} == "" ]] && { + echo "ERROR: ccm_db is not set" + exit 1 +} +if [[ "${groovy_script:-}" == "" ]]; then + echo "groovy_script must be set" + exit 1 +else + export groovy_script=$(pwd)/${groovy_script} + [[ -e ${groovy_script} ]] || { echo "ERROR: does not exist: $groovy_script" && exit 1; } +fi +[[ "${git_user_name:-}" == "" ]] && { echo "ERROR: git_user_name env variable must be set. It is used for the committer and init commit author" && exit 1; } +[[ "${git_user_email:-}" == "" ]] && { echo "ERROR: git_user_email env variable must be set. It is used for the committer and init commit author" && exit 1; } +[[ "${git_email_domain:-}" == "" ]] && { echo "ERROR: git_email_domain env variable must be set. It is used for the author domain. The username part of email is retrieved from CM/Synergy" && exit 1; } +[[ "${git_server_path:-}" == "" ]] && { echo "ERROR: git_server_path env variable must be set" && exit 1; } +[[ "${jiraProjectKey:-}" == "" ]] && { echo "ERROR: jiraProjectKey env variable must be set" && exit 1; } +#[[ "${use_cached_project_list:-}" == "" ]] && { export use_cached_project_list="true" ; } +echo "git_user_name=${git_user_name}" +echo "git_user_email=${git_user_email}" +echo "git_email_domain=${git_email_domain}" +echo "git_server_path=${git_server_path}" +echo "jiraProjectKey=${jiraProjectKey}" + +if [[ "${my_workspace_root:-}" == "" ]]; then + echo "INFO: my_workspace_root is not set .. defaulting to $(pwd)" + export my_workspace_root=$(pwd) +else + echo "INFO: my_workspace_root=${my_workspace_root}" +fi +if [[ -e $my_workspace_root ]] ; then + if git -C $my_workspace_root rev-parse --git-dir 2> /dev/null ; then + echo "git rev-parse --git-dir show that my_workspace_root=${my_workspace_root} is inside a repo - not supported" + echo "If you want to store it inside a Jenkins workspace then consider to checkout repo to a subdir and use \${WORKSPACE} as my_workspace_root" + exit 1 + else + echo "INFO: All good .. my_workspace_root = $my_workspace_root exists, but is not inside a git repo" + fi +fi + +source "$(pwd)/${BASH_SOURCE%/*}/_ccm-start-stop-functions.sh" || source ./_ccm-start-stop-functions.sh +ccm-start ${ccm_db} +source "$(pwd)/${BASH_SOURCE%/*}/_ccm-functions.sh" || source ./_ccm-functions.sh + +[[ "${debug:-}" == "true" ]] && set -x + +whoami +env > env.env + +if ! which java ; then + if [[ ${JAVA_HOME:-} != "" ]]; then + export PATH="${JAVA_HOME}/bin:$PATH" + which java || { echo "ERROR: java not found in PATH nor via $JAVA_HOME/bin" ; exit 1; } + echo "INFO: prepending $JAVA_HOME/bin to PATH" + else + echo "ERROR: java not found in PATH. Please set it in PATH or set JAVA_HOME and it is set to JAVA_HOME/bin" + exit 1 + fi +fi +java -version + +trap ccm-stop EXIT + +[[ ${ccm_project_name:-} == "" ]] && { echo "ERROR: please set variable: ccm_project_name with option of [:]" ; exit 1; } +export ccm_project_name_wo_instance=$(echo ${ccm_project_name} | awk -F ":" '{print $1}') + +export ccm_project_instance=$(echo ${ccm_project_name} | awk -F ":" '{print $2}') +if [[ ${ccm_project_instance:-} == "" ]]; then + if [[ ${ccm_dcm_dbid:-} != "" ]]; then + ccm_project_instance="${ccm_dcm_dbid}#1" + else + ccm_project_instance=1 + fi +fi + +export ccm_project_name_orig="" + +byref_translate_from_git_repo2ccm_name $ccm_project_name_wo_instance $ccm_project_instance ccm_project_name_orig + +rm -rf git2git_params.env && touch git2git_params.env +if [[ "${wipe_repo_before:-}" == "true" ]] ; then + echo "Execute: wipe_repo_before" + rm -rf ${my_workspace_root}/${ccm_project_name_wo_instance}/repo + echo "execute_mode=reclone" >> git2git_params.env +else + echo "INFO: wipe_repo_before is empty [''|false|true]" +fi +if [[ "${wipe_checkout_workspace_before:-}" == "true" ]] ; then + echo "Execute: wipe_checkout_workspace_before" + rm -rf ${my_workspace_root}/${ccm_project_name_wo_instance}/ccm_wa +else + echo "INFO: wipe_checkout_workspace_before is empty [''|false|true]" +fi + +if [[ "${tag_to_be_removed:-}" != "" ]] ; then + echo "tag_to_be_removed=${tag_to_be_removed}" >> git2git_params.env + cd ${my_workspace_root}/${ccm_project_name_wo_instance}/repo/${ccm_project_name_wo_instance} + git push origin $tag_to_be_removed --delete || echo "Deleting failed - skip" + git push ssh://git@${git_server_path}/${ccm_project_name_wo_instance}_orig.git $tag_to_be_removed --delete || echo "Deleting failed - skip" + if [[ ${git_server_path_prod_ccm2git:-} != "" ]]; then + git push origin $tag_to_be_removed --delete || echo "Deleting failed - skip" + git push ssh://git@${git_server_path_prod_ccm2git}/${ccm_project_name_wo_instance}_orig.git $tag_to_be_removed --delete || echo "Deleting failed - skip" + fi + git tag --delete $tag_to_be_removed + cd ${WORKSPACE} +else + echo "INFO: tag_to_be_removed: is not set " +fi + +[[ ${ccm_proj_rev_exclusion_query:-} == "" ]] && { echo "INFO: ccm_proj_rev_exclusion_query is not set" ; } + +rm -rf project_baselines.txt && touch project_baselines.txt +echo "INFO: Finding the $ccm_project_name_orig revisions that has has_no_baseline_project" +ccm query "\ + type='project' \ + and name='${ccm_project_name_orig}' \ + and instance='${ccm_project_instance}' \ + and ( status='integrate' or status='test' or status='sqa' or status='released' ) \ + and has_no_baseline_project() ${ccm_proj_rev_exclusion_query:-}" \ + -u -f "%objectname" \ + >> project_baselines.txt || { + exit_code=$? + if [[ $exit_code -eq 6 ]]; then + echo "Empty result: has_no_baseline_project - never mind" + elif [[ $exit_code -ne 0 ]] ; then + echo "ERROR: has_no_baseline_project something went wrong - exit : $exit_code" + exit $exit_code + fi + } + echo "" >> project_baselines.txt +echo "INFO: Finding the $ccm_project_name_orig which is is_hist_leaf" +bash "$(pwd)/${BASH_SOURCE%/*}/ccm-baseline-history-get-root.sh" \ + "$( ccm query "\ + type='project' \ + and name='${ccm_project_name_orig}' \ + and instance='${ccm_project_instance}' \ + and ( status='integrate' or status='test' or status='sqa' or status='released' ) \ + and is_hist_leaf() " \ + -u -f "%objectname" | head -1 \ + )" >> project_baselines.txt || { + exit_code=$? + if [[ $exit_code -eq 6 ]]; then + echo "Empty result: is_hist_leaf - never mind" + elif [[ $exit_code -ne 0 ]] ; then + echo "ERROR: is_hist_leaf something went wrong - exit : $exit_code" + exit $exit_code + fi + } + echo "" >> project_baselines.txt + +if [[ -f project_baselines.txt ]]; then + ccm_project_baselines=$( sort -u < project_baselines.txt ) + export ccm_project_baselines +else + echo "No project revision found - Neither from 'no_baseline' nor from history traverse' - exit 10" + exit 10 +fi + +IFS=$'\r\n' +for ccm_project_baseline in $( sort -u < project_baselines.txt ) ; do + + [[ "${ccm_project_baseline}" =~ ${regex_ccm4part:?} ]] || { + echo "4part does not comply" + return 1 + } + revision=${BASH_REMATCH[2]} + + git_repo_4part="" + byref_translate_from_ccm_4part2git_repo_4part "${ccm_project_baseline}" git_repo_4part + + [[ -d ${my_workspace_root}/${ccm_project_name_wo_instance}/ccm_wa/ ]] && touch ${my_workspace_root}/${ccm_project_name_wo_instance}/ccm_wa/projects.txt + [[ -d ${my_workspace_root}/${ccm_project_name_wo_instance}/repo/ ]] && touch ${my_workspace_root}/${ccm_project_name_wo_instance}/repo/git_sizes.txt + + cd ${BASH_SOURCE%/*}/ + echo "calling: java -jar build/libs/*.jar in $(pwd)" + set -x + java -jar build/libs/*.jar \ + ${groovy_script} \ + start_project="${git_repo_4part}" \ + my_workspace_root=${my_workspace_root} \ + git_server_path=${git_server_path} \ + jiraProjectKey=${jiraProjectKey} + [[ "${debug:-}" == "true" ]] && set -x # restore debug setting + cd - + cp "${my_workspace_root}/${ccm_project_name_wo_instance}/ccm_wa/projects.txt" "${my_workspace_root}/${ccm_project_name_wo_instance}/ccm_wa/projects-${revision}.txt" + cp "${my_workspace_root}/${ccm_project_name_wo_instance}/ccm_wa/projects-${revision}.txt" "${WORKSPACE}/" +done +cat projects-*.txt | sort -u > projects.txt +for file in $(ls -1 -rt ${my_workspace_root}/${ccm_project_name_wo_instance}/repo/*@git_size.txt); do + size=$(cat $file | awk -F" " '{print $1}') + echo "${size} @@@ ${file}" >> git_sizes.txt +done +ccm_amount_of_versions=$( wc -l < git_sizes.txt ) +git_size=$(tail -1 git_sizes.txt | awk -F " " '{print $1}') + +{ + echo "ccm_amount_of_versions=${ccm_amount_of_versions}" + echo "git_size=${git_size}" + echo "ccm_project_name_wo_instance=${ccm_project_name_wo_instance}" + echo "ccm_project_instance=${ccm_project_instance}" +} > ccm.env diff --git a/execute-ccm-git2git.sh b/execute-ccm-git2git.sh new file mode 100644 index 0000000..06bbec7 --- /dev/null +++ b/execute-ccm-git2git.sh @@ -0,0 +1,45 @@ +#!/bin/bash --login +set -x +set -u +set -e +set -o pipefail + +export ccm_project_name_wo_instance=$(echo ${ccm_project_name} | awk -F ":" '{print $1}') +export ccm_project_instance=$(echo ${ccm_project_name} | awk -F ":" '{print $2}') + +if [ "${ccm_project_instance}x" == "x" ]; then + export ccm_project_instance="1" +fi + +if [[ "${tag_to_be_removed:-}" != "" ]] ; then + cd ${ccm_project_name_wo_instance} + git push ssh://git@${git_server_path}/${ccm_project_name_wo_instance}.git $tag_to_be_removed --delete || echo "Deleting failed - skip" + git push ssh://git@${git_server_path}/${ccm_project_name_wo_instance}_orig.git $tag_to_be_removed --delete || echo "Deleting failed - skip" + if [[ ${git_server_path_prod_git2git:-} != "" ]] ; then + echo "Also handle production" + git push ssh://git@${git_server_path_prod_git2git}/${ccm_project_name_wo_instance}.git $tag_to_be_removed --delete || echo "Deleting failed - skip" + git push ssh://git@${git_server_path_prod_git2git}/${ccm_project_name_wo_instance}_orig.git $tag_to_be_removed --delete || echo "Deleting failed - skip" + fi + git tag --delete $tag_to_be_removed + cd ${WORKSPACE} +fi + +export PATH=${ccm_home_path}/bin:$PATH +export CCM_HOME=${ccm_home_path} + +which ccm +export CCM_ADDR=`ccm start -m -d /data/ccmdb/${ccm_db} -s ${ccm_server} -q` +[[ $CCM_ADDR == "" ]] && ( echo "CM/Synergy start failed" && exit 10 ) + +debug=true ${WORKSPACE}/ccm-convert-flat2submodules-existingRepoHistory.sh \ + "${ccm_project_name_wo_instance}" \ + "init" \ + "${repo_submodules}" \ + "${git_server_project}" \ + "${ccm_project_instance}" \ + "${gitignore_path_n_files:-}" \ + "${gitattributes_path_n_files:-}" + +echo "ccm_project_name_wo_instance=${ccm_project_name_wo_instance}" > ccm.env +echo "ccm_project_instance=${ccm_project_instance}" >> ccm.env + diff --git a/get_project_rev_count.sh b/get_project_rev_count.sh new file mode 100644 index 0000000..4013e88 --- /dev/null +++ b/get_project_rev_count.sh @@ -0,0 +1,12 @@ +#!/bin/bash --login +set -e +set -u +[[ ${debug:-} == true ]] && set -x + +IFS=$'\r\n' +{ + for proj in $(ccm query "type='project' and (status='integrate' or status='released') " -u -f "%name" | sort -u ) ; do + rev_count=$(ccm query "type='project' and name='${proj}' and ( status='released' or status='integrate' and status='test' and status='sqa')" -u -f "%objectname" | wc -l ) + printf "%s : %s\n" "$rev_count" "$proj" + done +} | sort -rh \ No newline at end of file diff --git a/git-fill-empty-dirs-with-gitignore.sh b/git-fill-empty-dirs-with-gitignore.sh new file mode 100644 index 0000000..6e6a645 --- /dev/null +++ b/git-fill-empty-dirs-with-gitignore.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -u +set -e +set -o pipefail +set -o posix + +[[ ${debug:-} == "true" ]] && set -x + +script_dir=$(dirname $(readlink -f $0)) +source $(dirname $0)/_git-functions.sh || source ./_git-functions.sh + +cd $1 +git_find_n_fill_empty_dirs_gitignore diff --git a/git-find-n-move-tag.sh b/git-find-n-move-tag.sh new file mode 100644 index 0000000..0b5a770 --- /dev/null +++ b/git-find-n-move-tag.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +[[ ${debug:-} == "true" ]] && set -x +set -u +set -e +set -o pipefail + +HEAD_of_tree=HEAD + +for sha1 in $(git rev-list ${HEAD_of_tree} ) ; do + tag_source=$(git tag --points-at $sha1) || { echo $sha1: null ; continue ; } + echo $sha1: $tag_source +done +for sha1 in $( git rev-list ${HEAD_of_tree} ) ; do + tag_source=$(git tag --points-at $sha1) + commit_line=$(git show -s --format=@%cd@%s refs/tags/${tag_source}^{}) + + sha1_of_target=$(git log --format=%H@%cd@%s origin/master | grep -e "${commit_line}" | cut -d @ -f 1 ) || { echo "ERROR: $tag_source commit is not found on origin/master" ; exit 1; } + + if [[ $(wc -w <<< ${sha1_of_target} ) -gt 1 ]]; then + echo "ERROR: $sha1_of_target have more than one sha1" + fi + printf "%s %s\n" "${tag_source}" "${sha1_of_target}" + + tag_target=${tag_source} + tag_old_source="old/${tag_source}" + + # reset the committer to get the correct set for the commiting the tag. There is no author of the tag + export GIT_AUTHOR_DATE=$(git tag -l --format="%(taggerdate:iso8601)" "${tag_source}" | awk -F" " '{print $1 " " $2}') && [[ -z ${GIT_AUTHOR_DATE} ]] && return 1 + export GIT_COMMITTER_DATE=${GIT_AUTHOR_DATE} + export GIT_COMMITTER_NAME=$(git tag -l --format="%(taggername)" "${tag_source}" ) && [[ -z ${GIT_COMMITTER_NAME} ]] && return 1 + export GIT_COMMITTER_EMAIL=$(git tag -l --format="%(taggeremail)" "${tag_source}" ) && [[ -z ${GIT_COMMITTER_EMAIL} ]] && return 1 + + echo "Get tag content of: ${tag_source}" + git tag -l --format '%(contents)' "${tag_source}" > ./tag_meta_data.txt + + echo "Tag the source tag for history reasons: ${tag_source} -> ${tag_old_source}" + git tag -a -F ./tag_meta_data.txt "${tag_old_source}" "${tag_source}^{}" + + echo "content of ${tag_source} to ${sha1_of_target}" + echo "git tag ${tag_target} based on ${tag_source}" + git tag -a -F ./tag_meta_data.txt "${tag_target}" "${sha1_of_target}" -f + rm -f ./tag_meta_data.txt +done diff --git a/git-remove-all-git-related-files-2plus-levels.sh b/git-remove-all-git-related-files-2plus-levels.sh new file mode 100644 index 0000000..6a60205 --- /dev/null +++ b/git-remove-all-git-related-files-2plus-levels.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -u +set -e +set -o pipefail +set -o posix + +[[ ${debug:-} == "true" ]] && set -x +cd $1 +pwd +echo "Files recursively named .gitignore .gitattributes .gitmodules" + +find . -mindepth 2 -type f -name '.gitignore' -o -name '.gitmodules' -o -name '.gitattributes' > remove_files.tmp || touch remove_files.tmp + +IFS=$'\n\r' +for file in $(cat remove_files.tmp) ; do + echo "remove file: $file" + [[ ${dryrun:-} == true ]] || rm -f "$file" +done + +rm -f remove_files.tmp diff --git a/git-set-execute-bit-in-index-of-extensions.sh b/git-set-execute-bit-in-index-of-extensions.sh new file mode 100644 index 0000000..e2e754f --- /dev/null +++ b/git-set-execute-bit-in-index-of-extensions.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -u +set -e +set -o pipefail +set -o posix + +[[ ${debug:-} == "true" ]] && set -x + +script_dir=$(dirname $(readlink -f $0)) +source $(dirname $0)/_git-functions.sh || source ./_git-functions.sh + +cd $1 +git_set_execute_bit_in_index_of_extensions diff --git a/git-set-execute-bit-in-index-of-unix-tool-file-executable.sh b/git-set-execute-bit-in-index-of-unix-tool-file-executable.sh new file mode 100644 index 0000000..9ea2189 --- /dev/null +++ b/git-set-execute-bit-in-index-of-unix-tool-file-executable.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -u +set -e +set -o pipefail +set -o posix + +[[ ${debug:-} == "true" ]] && set -x + +script_dir=$(dirname $(readlink -f $0)) +source $(dirname $0)/_git-functions.sh || source ./_git-functions.sh + +cd "$1" +exit_code=0 +git_set_execute_bit_in_index_of_unix_tool_file_executable || exit_code=$? +exit $exit_code diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index f6b961f..8bdaf60 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 16d2805..2a84e18 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.7-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-9.0.0-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index cccdd3d..ef07e01 100755 --- a/gradlew +++ b/gradlew @@ -1,78 +1,129 @@ -#!/usr/bin/env sh +#!/bin/sh + +# +# Copyright © 2015 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar +CLASSPATH="\\\"\\\"" + # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -81,92 +132,120 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=$((i+1)) + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" fi +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat old mode 100755 new mode 100644 index e95643d..5eed7ee --- a/gradlew.bat +++ b/gradlew.bat @@ -1,4 +1,22 @@ -@if "%DEBUG%" == "" @echo off +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -9,25 +27,29 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init +if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -35,48 +57,36 @@ goto fail set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe -if exist "%JAVA_EXE%" goto init +if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - :execute @rem Setup the command line -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar +set CLASSPATH= + @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal diff --git a/src/main/groovy/toGit/migration/sources/ccm/CCMSource.groovy b/src/main/groovy/toGit/migration/sources/ccm/CCMSource.groovy new file mode 100644 index 0000000..efd64b6 --- /dev/null +++ b/src/main/groovy/toGit/migration/sources/ccm/CCMSource.groovy @@ -0,0 +1,164 @@ +package toGit.migration.sources.ccm + +import org.apache.commons.io.FileUtils +import org.slf4j.LoggerFactory + +import toGit.context.base.Context +import toGit.migration.plan.Criteria +import toGit.migration.plan.Snapshot +import toGit.migration.sources.MigrationSource +import toGit.migration.sources.ccm.context.CcmCriteriaContext +import toGit.migration.sources.ccm.context.CcmExtractionsContext + +class CCMSource implements MigrationSource { + + final static log = LoggerFactory.getLogger(this.class) + + String revision + String proj_instance + String name4part + String ccm_addr + String ccm_home + String system_path + String jiraProjectKey + + @Override + List getSnapshots(List initialFilter) { + + List projects + + // Build the CCM project conversion list + def sout = new StringBuilder(), serr = new StringBuilder() + def cmd_line = ["bash", System.getProperty("user.dir") + File.separator + "ccm-baseline-history.sh", "${name4part}"] + + log.info cmd_line.toString() + + def envVars = System.getenv().collect { k, v -> "$k=$v" } + def cmd = cmd_line.execute(envVars,new File(workspace)) + cmd.waitForProcessOutput(sout, serr) + def exitValue = cmd.exitValue() + + println (sout.toString()) + if ( exitValue ){ + println ("Standard error output" ) + println (serr.toString()) + throw new Exception("ccm-baseline-history gave an non-0 exit code" ) + } + if ( serr.toString().readLines().size() > 0 ){ + println ("Standard error output - used for SKIP projects" ) + println (serr.toString()) + } + + projects = sout.readLines().collect{new Snapshot(it){}} + + log.info projects.size().toString() + + return projects + } + + @Override + void checkout(Snapshot snapshot) { + copy2Filesystem(snapshot) + } + + private void copy2Filesystem(Snapshot snapshot) { + def gitSnapshotThis = snapshot.identifier.split("@@@")[0] + def gitSnapshotBaseline = snapshot.identifier.split("@@@")[1] + def gitSnapshotName = gitSnapshotThis.split(System.getenv("ccm_delim"))[0] + def gitSnapshotRevision = gitSnapshotThis.split(System.getenv("ccm_delim"))[1].split(":")[0] + def gitSnapshotInstance = gitSnapshotThis.split(System.getenv("ccm_delim"))[1].split(":")[2] + def gitBaselineRevision = gitSnapshotBaseline.split(System.getenv("ccm_delim"))[1].split(":")[0] + def gitBaselineInstance = gitSnapshotBaseline.split(System.getenv("ccm_delim"))[1].split(":")[2] + + def ccmSnapshotThis = snapshot.identifier.split("@@@")[2] + def ccmSnapshotName = ccmSnapshotThis.split(System.getenv("ccm_delim"))[0] + def ccmSnapshotBaseline = snapshot.identifier.split("@@@")[3] + + def gitSnapshot_revision_for_ws=gitSnapshotThis.split(":")[0] + + def codeFile = new File(workspace, "code") + codeFile.parentFile.mkdirs() + if ( ! codeFile.exists()) { + codeFile.delete() + } + codeFile.mkdir() + + + def path_final=workspace + "/code/" + gitSnapshot_revision_for_ws + def file_full_path_name="${path_final}/" + gitSnapshotName + + if ( new File(file_full_path_name).exists()){ + log.info "CM/Synergy checkout: Skipping project revision: ${gitSnapshot_revision_for_ws} - already exists" + } else { + def sout = new StringBuilder(), serr = new StringBuilder() + def path_tmp="${path_final}_tmp" + + def file_tmp = new File(path_tmp) + if ( file_tmp.exists() ){ + log.info "${path_tmp} exist - Delete it " + file_tmp.deleteDir() + } + + def file_full_path_spaced_name = new File ("${path_final}/" + ccmSnapshotThis.split( System.getenv("ccm_delim") )[0]) + if ( file_full_path_spaced_name.exists() ) { + log.info file_full_path_spaced_name.toString() + " exist due to previous error - Delete it all" + def file_base = new File (path_final) + file_base.deleteDir() + } + + def envVars = System.getenv().collect { k, v -> "$k=$v" } + def cmd_line = ["ccm", "copy_to_file_system", "-p", "${gitSnapshot_revision_for_ws}_tmp", "-r", "${ccmSnapshotThis}"] + log.info "'" + cmd_line + "'" + def cmd = cmd_line.execute(envVars, codeFile) + cmd.waitForProcessOutput(sout, serr) + def exitValue = cmd.exitValue() + log.info "Standard out:" + log.info "'" + sout + "'" + log.info "Standard error:" + log.info "'" + serr + "'" + log.info "Exit code: " + exitValue + if ( exitValue ){ + throw new Exception("ccm copy_to_file_system gave an non-0 exit code" ) + } + if ( serr.toString().readLines().size() > 0 ){ + throw new Exception("ccm copy_to_file_system standard error contains text lines: " + serr.toString().readLines().size() ) + } + + if ( ! new File(path_tmp + "/" + ccmSnapshotName ).exists() ) { + if ( ! new File(path_tmp).exists() ) { + log.info "Checkout is empty - make an empty dir: " + path_final + new File(path_tmp).mkdir() + } + log.info "Checkout is empty - make an empty dir: " + path_final + "/" + ccmSnapshotName + "/" + ccmSnapshotName + new File(path_tmp + "/" + ccmSnapshotName).mkdir() + } + log.info "Move from: ${path_tmp} to: ${path_final}" + FileUtils.moveDirectory(new File(path_tmp), new File(path_final)) + + if ( gitSnapshotName != ccmSnapshotName ){ + log.info "ccm and git names differ.. move ccm name to git name: " + path_final + "/" + ccmSnapshotName + " -> " + file_full_path_name + FileUtils.moveDirectory(new File(path_final + "/" + ccmSnapshotName), new File(file_full_path_name)) + } + } + } + + @Override + void prepare() { + //Copy utility scripts to temporary folder during migration from resources + } + + @Override + void cleanup() { + + } + + @Override + Context withCriteria(Context criteriaContext) { + return criteriaContext as CcmCriteriaContext + } + + @Override + Context withExtractions(Context extractionsContext) { + return extractionsContext as CcmExtractionsContext + } +} diff --git a/src/main/groovy/toGit/migration/sources/ccm/CCMSourceContext.groovy b/src/main/groovy/toGit/migration/sources/ccm/CCMSourceContext.groovy new file mode 100644 index 0000000..bf2587b --- /dev/null +++ b/src/main/groovy/toGit/migration/sources/ccm/CCMSourceContext.groovy @@ -0,0 +1,68 @@ +package toGit.migration.sources.ccm + +import toGit.context.base.Context +import toGit.context.traits.SourceContext + + +class CCMSourceContext implements Context, SourceContext { + public CCMSourceContext() { + source = new CCMSource() + } + + void workspace(String path) { + source.workspace = path + } + + void revision(String revision){ + source.revision = revision + } + + @Deprecated + void name4part (String name4part){ + namePart(name4part) + } + + void namePart(String name4part){ + source.name4part = name4part + } + + @Deprecated + void proj_instance(String proj_instance) { + projectInstance(proj_instance) + } + + void projectInstance(String proj_instance){ + source.proj_instance = proj_instance + } + + @Deprecated + void ccm_addr (String ccm_addr){ + address(ccm_addr) + } + + void address(String ccm_addr){ + source.ccm_addr = ccm_addr + } + + @Deprecated + void ccm_home(String ccm_home){ + home(ccm_home) + } + + void home (String ccm_home){ + source.ccm_home = ccm_home + } + + @Deprecated + void system_path(String system_path){ + systemPath(system_path) + } + + void systemPath(String system_path){ + source.system_path = system_path + } + + void jiraProjectKey(String jiraProjectKey){ + source.jiraProjectKey = jiraProjectKey + } +} diff --git a/src/main/groovy/toGit/migration/sources/ccm/context/CcmCriteriaContext.groovy b/src/main/groovy/toGit/migration/sources/ccm/context/CcmCriteriaContext.groovy new file mode 100644 index 0000000..881db5f --- /dev/null +++ b/src/main/groovy/toGit/migration/sources/ccm/context/CcmCriteriaContext.groovy @@ -0,0 +1,21 @@ +package toGit.migration.sources.ccm.context + +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import toGit.context.base.Context + +import toGit.migration.sources.ccm.criteria.AlreadyConverted + +trait CcmCriteriaContext implements Context { + final static Logger log = LoggerFactory.getLogger(this.class) + + + /** + * Filters out baselines that don't have given promotion levels + * @param levels the promotion levels to test baselines against + */ + void AlreadyConverted(String repo_path) { + criteria.add(new AlreadyConverted(repo_path)) + log.debug("Added 'AlreadyConverted' criteria.") + } +} diff --git a/src/main/groovy/toGit/migration/sources/ccm/context/CcmExtractionsContext.groovy b/src/main/groovy/toGit/migration/sources/ccm/context/CcmExtractionsContext.groovy new file mode 100644 index 0000000..588c256 --- /dev/null +++ b/src/main/groovy/toGit/migration/sources/ccm/context/CcmExtractionsContext.groovy @@ -0,0 +1,19 @@ +package toGit.migration.sources.ccm.context + +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import toGit.context.base.Context +import toGit.migration.sources.ccm.extractions.MetaBaseline + +trait CcmExtractionsContext implements Context { + final static Logger log = LoggerFactory.getLogger(this.class) + + /** + * Extracts a CoolBaseline property + * @param map A map of values to extract and keys to map them to. + */ + void baselineProperties(String ccm_workspace, String jiraProjectKey) { + extractions.add(new MetaBaseline(ccm_workspace, jiraProjectKey)) + log.debug("Added 'baselineProperties' criteria.") + } +} diff --git a/src/main/groovy/toGit/migration/sources/ccm/criteria/AlreadyConverted.groovy b/src/main/groovy/toGit/migration/sources/ccm/criteria/AlreadyConverted.groovy new file mode 100644 index 0000000..deeb252 --- /dev/null +++ b/src/main/groovy/toGit/migration/sources/ccm/criteria/AlreadyConverted.groovy @@ -0,0 +1,66 @@ +package toGit.migration.sources.ccm.criteria + +import org.slf4j.LoggerFactory +import toGit.migration.plan.Criteria +import toGit.migration.plan.Snapshot + + +class AlreadyConverted extends Criteria { + + final static log = LoggerFactory.getLogger(this.class) + + String repo_path + def tags = [] + + AlreadyConverted(String repo_path){ + this.repo_path = repo_path + if ( new File(repo_path + File.separator + ".git" ).exists() ) { + + def sout = new StringBuilder(), serr = new StringBuilder() + def cmd_line = "git tag" + log.info cmd_line + " : in path: " + repo_path + + def envVars = [] + def cmd_process = cmd_line.execute(envVars,new File(repo_path)) + cmd_process.waitForProcessOutput(sout, serr) + + def exitValue = cmd_process.exitValue() + if ( exitValue ) { + throw new Exception('WHAT??') + } + + log.info "Read the tags from the git tag output" + sout.eachLine { line -> + tags << line + } + log.info "Done" + + tags.each { + log.info it + } + } else { + log.info ('.git not available - assuming the repo is not initialized yet - skip') + } + } + + @Override + boolean appliesTo(Snapshot snapshot) { + log.info ("Entering (ccm2git) appliesTo") + def snapshotRevision = snapshot.identifier.split("@@@")[0].split( System.getenv("ccm_delim") )[1].split(":")[0] + + def tag_regex = snapshotRevision.replace('+','\\+') + "_" + "[dprtis][eueenq][lblsta]\$" + + def convert = true + tags.each { tag -> + if ( tag ==~ /$tag_regex/ ) { + log.info ("Already converted - skip: " + snapshot.identifier + " ~ " + tag_regex ) + convert = false + return + } + } + if ( convert ) { + log.info ("Not converted - do it: " + snapshot.identifier + " ~ " + tag_regex ) + } + return convert + } +} \ No newline at end of file diff --git a/src/main/groovy/toGit/migration/sources/ccm/extractions/MetaBaseline.groovy b/src/main/groovy/toGit/migration/sources/ccm/extractions/MetaBaseline.groovy new file mode 100644 index 0000000..e57c425 --- /dev/null +++ b/src/main/groovy/toGit/migration/sources/ccm/extractions/MetaBaseline.groovy @@ -0,0 +1,202 @@ +package toGit.migration.sources.ccm.extractions + +import org.slf4j.LoggerFactory +import toGit.migration.plan.Extraction +import toGit.migration.plan.Snapshot +import toGit.migration.sources.ccucm.Baseline + +class MetaBaseline extends Extraction { + + final static log = LoggerFactory.getLogger(this.class) + + String workspace + String jiraProjectKey + + MetaBaseline(String ccm_workspace, String jiraProjectKey) { + this.workspace = ccm_workspace + this.jiraProjectKey = jiraProjectKey + } + + @Override + HashMap extract(Snapshot snapshot ) { + def result = [:] + + def ccm_delimiter = System.getenv("ccm_delim") + + def gitSnapshotThis = snapshot.identifier.split("@@@")[0] + def gitSnapshotBaseline = snapshot.identifier.split("@@@")[1] + def gitSnapshotName = gitSnapshotThis.split( ccm_delimiter )[0] + def gitSnapshotRevision = gitSnapshotThis.split( ccm_delimiter )[1].split(":")[0] + def gitSnapshotInstance = gitSnapshotThis.split( ccm_delimiter )[1].split(":")[2] + def gitBaselineRevision = gitSnapshotBaseline.split( ccm_delimiter )[1].split(":")[0] + def gitBaselineInstance = gitSnapshotBaseline.split( ccm_delimiter )[1].split(":")[2] + + def ccmSnapshotThis = snapshot.identifier.split("@@@")[2] + def ccmSnapshotBaseline = snapshot.identifier.split("@@@")[3] + def ccmSnapshotRevision = ccmSnapshotThis.split( ccm_delimiter )[1].split(":")[0] + def ccmSnapshotInstance = ccmSnapshotThis.split( ccm_delimiter )[1].split(":")[2] + + result['gitSnapshot'] = gitSnapshotThis + result['gitSnapshotName'] = gitSnapshotName + result['gitSnapshotRevision'] = gitSnapshotRevision + result['gitSnapshotInstance'] = gitSnapshotInstance + result['ccmSnapshotThis'] = ccmSnapshotThis + result['ccmSnapshotBaseline'] = ccmSnapshotBaseline + result['ccmSnapshotRevision'] = ccmSnapshotRevision + result['ccmSnapshotInstance'] = ccmSnapshotInstance + + def envVars = System.getenv().collect { k, v -> "$k=$v" } + def cmd_line + def cmd + def exitValue + def sout + def serr + sout = new StringBuilder() + serr = new StringBuilder() + + // get baseline or project baseline status of the baseline revision + if ( gitBaselineRevision ==~ /init/ ) { + // run this sections of we are pointing to init + result['gitBaselineRevision'] = gitBaselineRevision + result['gitBaselineRevision_wstatus'] = gitBaselineRevision + } else { + // run this sections of we are pointing to any revision other than init alias an already converted + cmd_line = ["bash", System.getProperty("user.dir") + File.separator + "ccm-get-status-from-baseline-or-project.sh", "${ccmSnapshotBaseline}"] + + log.info(cmd_line) + cmd = cmd_line.execute(envVars, new File(workspace)) + cmd.waitForProcessOutput(sout, serr) + exitValue = cmd.exitValue() + log.info "stdout: " + sout.toString().trim() + if (exitValue) { + log.error "Standard error:" + log.error "'" + serr + "'" + log.error "Exit code: " + exitValue + throw new Exception(cmd_line + " gave exit code: $exitValue") + } + if (serr.toString().readLines().size() > 0) { + log.error "Standard error:" + log.error "'" + serr + "'" + log.error "Exit code: " + exitValue + } + result['gitBaselineRevision'] = gitBaselineRevision + result['gitBaselineRevision_wstatus'] = gitBaselineRevision + '_' + sout.toString().trim() + sout = new StringBuilder() + serr = new StringBuilder() + } + // get the owner of the project revision + cmd_line = ["ccm", "attr", "-show", "owner", "${ccmSnapshotThis}"] + log.info "'" + cmd_line + "'" + cmd = cmd_line.execute(envVars,new File(workspace)) + cmd.waitForProcessOutput(sout, serr) + exitValue = cmd.exitValue() + log.info "Exit code: " + exitValue + log.info "Standard out:" + log.info "'" + sout.toString().trim() + "'" + if ( exitValue ){ + log.info "Standard error:" + log.info "'" + serr + "'" + throw new Exception("Get owner gave an non-0 exit code" ) + } + result['snapshotOwner'] = sout.toString().trim() + sout = new StringBuilder() + serr = new StringBuilder() + + // Get the baseline date from project + cmd_line = ["ccm", "properties", "-f", "\"%{create_time[dateformat='yyyy-MM-dd HH:MM:SS']}\"", ccmSnapshotThis ] + log.info "'" + cmd_line + "'" + cmd = cmd_line.execute(envVars,new File(workspace)) + cmd.waitForProcessOutput(sout, serr) + exitValue = cmd.exitValue() + log.info "stdout: " + sout.toString().trim() + if ( exitValue ){ + log.error "Standard error:" + log.error "'" + serr + "'" + log.error "Exit code: " + exitValue + throw new Exception(cmd_line.toString() + ": gave an non-0 exit code" ) + } + if ( serr.toString().readLines().size() > 0 ){ + log.error "Standard error:" + log.error "'" + serr + "'" + log.error "Exit code: " + exitValue + } + result['snapshot_commiter_date'] = sout.toString().trim() + sout = new StringBuilder() + serr = new StringBuilder() + + // get baseline or project baseline status from + cmd_line = ["bash", System.getProperty("user.dir") + File.separator + "ccm-get-status-from-baseline-or-project.sh", "${ccmSnapshotThis}"] + + log.info(cmd_line) + cmd = cmd_line.execute(envVars,new File(workspace)) + cmd.waitForProcessOutput(sout, serr) + exitValue = cmd.exitValue() + log.info "stdout status: " + sout.toString().trim() + if ( exitValue ){ + log.error "Standard error:" + log.error "'" + serr + "'" + log.error "Exit code: " + exitValue + throw new Exception(cmd_line +": gave exit code: $exitValue" ) + } + if ( serr.toString().readLines().size() > 0 ){ + log.error "Standard error:" + log.error "'" + serr + "'" + log.error "Exit code: " + exitValue + } + result['snapshot_status'] = sout.toString().trim() + sout = new StringBuilder() + serr = new StringBuilder() + + // Build the CCM project meta data for later commit + cmd_line = ["bash", + System.getProperty("user.dir") + File.separator + "ccm-extract-baseline-project-metadata.sh", + "${ccmSnapshotThis}", + jiraProjectKey, + "tag" + ] + log.info cmd_line.toString() + cmd = cmd_line.execute(envVars,new File(workspace)) + cmd.waitForProcessOutput(sout, serr) + exitValue = cmd.exitValue() + log.info "stdout lines count: " + sout.toString().readLines().size() + if ( exitValue ){ + log.error "Standard error:" + log.error "'" + serr + "'" + log.error "Exit code: " + exitValue + throw new Exception( cmd_line + ": gave exit code: $exitValue" ) + } + if ( serr.toString().readLines().size() > 0 ){ + log.error "Standard error:" + log.error "'" + serr + "'" + log.error "Exit code: " + exitValue + } + result['baseline_tag_info'] = sout + + sout = new StringBuilder() + serr = new StringBuilder() + + cmd_line = ["bash", System.getProperty("user.dir") + File.separator + "ccm-extract-baseline-project-metadata.sh", + "${ccmSnapshotThis}", + jiraProjectKey, + "commit" + ] + log.info cmd_line.toString() + cmd = cmd_line.execute(envVars,new File(workspace)) + cmd.waitForProcessOutput(sout, serr) + exitValue = cmd.exitValue() + log.info "stdout lines count: " + sout.toString().readLines().size() + if ( exitValue ){ + log.error "Standard error:" + log.error "'" + serr + "'" + log.error "Exit code: " + exitValue + throw new Exception( cmd_line + ": gave exit code: $exitValue" ) + } + if ( serr.toString().readLines().size() > 0 ){ + log.error "Standard error:" + log.error "'" + serr + "'" + log.error "Exit code: " + exitValue + } + result['baseline_commit_info'] = sout + return result + } +} \ No newline at end of file diff --git a/src/main/groovy/toGit/migration/targets/git/actions/FillEmptyDirs.groovy b/src/main/groovy/toGit/migration/targets/git/actions/FillEmptyDirs.groovy new file mode 100644 index 0000000..bb5a831 --- /dev/null +++ b/src/main/groovy/toGit/migration/targets/git/actions/FillEmptyDirs.groovy @@ -0,0 +1,36 @@ +package toGit.migration.targets.git.actions + +import org.slf4j.LoggerFactory +import org.apache.commons.io.FileUtils +import toGit.migration.plan.Action + +class FillEmptyDirs extends Action { + + final static log = LoggerFactory.getLogger(this.class) + + String path + + public FillEmptyDirs(String path) { + this.path = path + } + + @Override + void act(HashMap extractionMap) { + log.info("Sprinkling dummy files in empty directories") + sprinkleDummies(new File(path)) + log.info("Finished sprinkling dummy files") + } + + def sprinkleDummies(File directory) { + def subDirs = directory.listFiles().findAll { it.isDirectory() && !it.name.equals(".git") } + subDirs.each { subDir -> + def contents = subDir.listFiles() + if(contents.any()) { + sprinkleDummies(subDir) + } else { + FileUtils.copyFileToDirectory(new File(System.getProperty("user.dir") + File.separator + "emptydir.gitignore"), targetDir) + log.info("Dropped .gitignore file in $subDir") + } + } + } +} diff --git a/src/main/groovy/toGit/migration/targets/git/actions/Setup.groovy b/src/main/groovy/toGit/migration/targets/git/actions/Setup.groovy new file mode 100644 index 0000000..f440e9a --- /dev/null +++ b/src/main/groovy/toGit/migration/targets/git/actions/Setup.groovy @@ -0,0 +1,32 @@ +package toGit.migration.targets.git.actions + +import org.slf4j.LoggerFactory +import toGit.migration.plan.Action +import toGit.migration.targets.git.GitOptions +import toGit.migration.targets.git.GitUtil + +class Setup extends Action { + + final static log = LoggerFactory.getLogger(this.class) + + File path + GitOptions options + + public Setup(File path, GitOptions options) { + this.path = path + this.options = options + } + + @Override + void act(HashMap extractionMap) { + if (!path.exists()) { + log.info("Initializing Git repository in $path") + GitUtil.initRepository(path) + GitUtil.configureRepository(path, options) + GitUtil.initCommit(path) + log.info("Initialized Git repository") + } else { + log.info("Git repository already exists in $path") + } + } +} \ No newline at end of file diff --git a/src/main/groovy/togit/ScriptBase.groovy b/src/main/groovy/togit/ScriptBase.groovy index f08679e..e12ec67 100644 --- a/src/main/groovy/togit/ScriptBase.groovy +++ b/src/main/groovy/togit/ScriptBase.groovy @@ -3,21 +3,23 @@ package togit import static togit.context.ContextHelper.executeInContext import org.slf4j.LoggerFactory -import org.apache.commons.lang.NotImplementedException -import togit.context.Context -import togit.context.SourceContext -import togit.context.TargetContext -import togit.context.DslContext -import togit.context.MigrationContext -import togit.migration.MigrationManager -import togit.migration.sources.MigrationSource -import togit.migration.sources.ccbase.context.ClearCaseSourceContext -import togit.migration.sources.ccucm.context.CcucmSourceContext -import togit.migration.sources.dummy.DummySourceContext -import togit.migration.targets.MigrationTarget -import togit.migration.targets.artifactory.context.ArtifactoryTargetContext -import togit.migration.targets.dummy.DummyTargetContext -import togit.migration.targets.git.context.GitTargetContext +import toGit.context.MigrationContext +import toGit.context.base.Context +import toGit.context.base.DslContext +import toGit.context.traits.SourceContext +import toGit.context.traits.TargetContext +import toGit.migration.MigrationManager +import toGit.migration.sources.MigrationSource +import toGit.migration.sources.ccbase.context.ClearCaseSourceContext +import toGit.migration.sources.ccucm.context.CcucmSourceContext +import toGit.migration.sources.ccm.CCMSourceContext +import toGit.migration.sources.dummy.DummySourceContext +import toGit.migration.targets.MigrationTarget +import toGit.migration.targets.artifactory.context.ArtifactoryTargetContext +import toGit.migration.targets.dummy.DummyTargetContext +import toGit.migration.targets.git.context.GitTargetContext + +import static toGit.context.ContextHelper.executeInContext /** * Script base for the DSL. @@ -28,10 +30,11 @@ abstract class ScriptBase extends Script implements Context { final static LOG = LoggerFactory.getLogger(this.class) // Supported sources - final HashMap sourceTypes = [ - 'dummy':DummySourceContext, - 'ccucm':CcucmSourceContext, - 'clearcase':ClearCaseSourceContext, + final Map sourceTypes = [ + 'dummy': DummySourceContext, + 'ccucm': CcucmSourceContext, + 'ccm': CCMSourceContext, + 'clearcase': ClearCaseSourceContext, ] // Supported targets diff --git a/src/main/groovy/togit/context/ActionsContext.groovy b/src/main/groovy/togit/context/ActionsContext.groovy index ebf795e..9e7dc06 100644 --- a/src/main/groovy/togit/context/ActionsContext.groovy +++ b/src/main/groovy/togit/context/ActionsContext.groovy @@ -29,14 +29,19 @@ class ActionsContext implements Context { actions.add(new Action() { @Override void act(HashMap extractionMap) { - File sourceDir = new File(source) - File targetDir = new File(target) + def expandedSource = new SimpleTemplateEngine().createTemplate(source).make(extractionMap).toString() + def expandedTarget = new SimpleTemplateEngine().createTemplate(target).make(extractionMap).toString() + def sourceDir = new File(expandedSource) + if (! sourceDir.exists() ) throw new Exception ("ERROR: Source directory:" + sourceDir + " does not exist") + def targetDir = new File(expandedTarget) + if (! targetDir.exists() ) throw new Exception ("ERROR: Target directory:" + targetDir + " does not exist") + log.info ("Copy from: $expandedSource") + log.info ("To - perserving dates: $expandedTarget") sourceDir.listFiles().each { file -> if (file.isDirectory()) { FileUtils.copyDirectoryToDirectory(file, targetDir) - } else { - FileUtils.copyFileToDirectory(file, targetDir) - } + else + FileUtils.copyFileToDirectory(file, targetDir, true ) } } }) @@ -90,9 +95,10 @@ class ActionsContext implements Context { actions.add(new Action() { @Override void act(HashMap extractionMap) { - String expandedCmd = new SimpleTemplateEngine().createTemplate(command).make(extractionMap) - CommandLine.newInstance().run(expandedCmd, path ? new File(path) : null).stdoutBuffer.eachLine { line -> - LOG.info(line) + def expandedCommand = new SimpleTemplateEngine().createTemplate(command).make(extractionMap).toString() + log.info(expandedCommand) + CommandLine.newInstance().run(expandedCommand, path ? new File(path) : null).stdoutBuffer.eachLine { line -> + log.info(line) } } }) diff --git a/src/main/groovy/togit/migration/targets/artifactory/ArtifactoryTarget.groovy b/src/main/groovy/togit/migration/targets/artifactory/ArtifactoryTarget.groovy index c134e0b..f17cf39 100644 --- a/src/main/groovy/togit/migration/targets/artifactory/ArtifactoryTarget.groovy +++ b/src/main/groovy/togit/migration/targets/artifactory/ArtifactoryTarget.groovy @@ -1,6 +1,6 @@ package togit.migration.targets.artifactory -import static org.jfrog.artifactory.client.ArtifactoryClient.create +import static org.jfrog.artifactory.client.ArtifactoryClientBuilder.create import org.jfrog.artifactory.client.Artifactory import org.slf4j.LoggerFactory @@ -33,7 +33,7 @@ class ArtifactoryTarget implements MigrationTarget { } Artifactory getClient() { - client = client ?: create(options.url, options.user, options.password) - client + if (!client) client = create().setUrl(options.url).setUser(options.user).setPassword(options.password).build() + return client } } diff --git a/src/main/groovy/togit/migration/targets/git/GitOptions.groovy b/src/main/groovy/togit/migration/targets/git/GitOptions.groovy index eac39aa..be3dd2a 100644 --- a/src/main/groovy/togit/migration/targets/git/GitOptions.groovy +++ b/src/main/groovy/togit/migration/targets/git/GitOptions.groovy @@ -6,6 +6,9 @@ class GitOptions { String user = 'migration' String email = '2git@praqma.com' String remote = '' + String initCommitDate = "1970-01-01 00:00" + + String jiraProjectKey = "" boolean defaultSetup = true boolean longPaths = false diff --git a/src/main/groovy/togit/migration/targets/git/GitUtil.groovy b/src/main/groovy/togit/migration/targets/git/GitUtil.groovy index 27831db..2783ae1 100644 --- a/src/main/groovy/togit/migration/targets/git/GitUtil.groovy +++ b/src/main/groovy/togit/migration/targets/git/GitUtil.groovy @@ -46,21 +46,20 @@ class GitUtil { */ static void configureRepository(File path, GitOptions options) { if (options.user) { - callOrDie(path, 'config', 'user.name', options.user) - LOG.debug("Set git user.name to $options.user.") + callOrDie(path, "config", "user.name", "\"$options.user\"") + log.debug("Set git user.name to $options.user.") } if (options.email) { - callOrDie(path, 'config', 'user.email', options.email) - LOG.debug("Set git user.email to $options.user.") + callOrDie(path, "config", "user.email", options.email) + log.debug("Set git user.email to $options.email.") } if (options.longPaths) { callOrDie(path, 'config', 'core.longpaths', 'true') } - - if (options.remote) { - call(path, 'remote', 'add', 'origin', options.remote) + if (!(("").equals(options.remote))) { + callOrDie(path, "remote", "add", "origin", options.remote) } writeGitIgnore(path, options) @@ -69,7 +68,8 @@ class GitUtil { static void setGitLfs(File path, GitOptions options) { options.lfs.each { file -> - callOrDie(path, 'lfs', 'track', "'$file'") + callOrDie(path, "lfs", "track", "'$file'") + callOrDie(path, "add", ".gitattributes" ) } } @@ -81,6 +81,11 @@ class GitUtil { options.ignore.each { rule -> FileUtils.writeStringToFile(gitIgnore, rule + '\n', true) } + log.info(".gitignore created") + if (gitIgnore.exists()) + log.info(".gitignore add") + callOrDie(path, "add", ".gitignore") + log.info(".gitignore created") } /** @@ -106,4 +111,74 @@ class GitUtil { callOrDie(path, 'init') } } + + static void initCommit(File path) { + if (!path.exists()) { + log.error("Git dir $path does not exist: FAIL") + } else { + + log.info("Creating the init commit") + def sout = new StringBuilder(), serr = new StringBuilder() + def cmd + def cmd_line + log.info("Setting environment: GIT_COMMITTER_DATE, GIT_AUTHOR_DATE") + def envVars = System.getenv().collect { k, v -> "$k=$v" } + envVars.add('GIT_COMMITTER_DATE=1970-01-01 11:11:11') + envVars.add('GIT_AUTHOR_DATE=1970-01-01 11:11:11') + + // Create the init commit + cmd_line = "git commit --allow-empty -m init" + log.debug("Executing '$cmd_line' in $path") + cmd = cmd_line.execute(envVars,path) + cmd.waitForProcessOutput(sout, serr) + println sout + println serr + + // Create the tag + cmd_line = "git tag -m init init" + log.debug("Executing '$cmd_line' in $path") + cmd = cmd_line.execute(envVars,path) + cmd.waitForProcessOutput(sout, serr) + + println sout + println serr + + + } + } + + static void initCommit(File path) { + if (!path.exists()) { + log.error("Git dir $path does not exist: FAIL") + } else { + + log.info("Creating the init commit") + def sout = new StringBuilder(), serr = new StringBuilder() + def cmd + def cmd_line + log.info("Setting environment: GIT_COMMITTER_DATE, GIT_AUTHOR_DATE") + def envVars = System.getenv().collect { k, v -> "$k=$v" } + envVars.add('GIT_COMMITTER_DATE=1970-01-01 11:11:11') + envVars.add('GIT_AUTHOR_DATE=1970-01-01 11:11:11') + + // Create the init commit + cmd_line = "git commit --allow-empty -m init" + log.debug("Executing '$cmd_line' in $path") + cmd = cmd_line.execute(envVars,path) + cmd.waitForProcessOutput(sout, serr) + println sout + println serr + + // Create the tag + cmd_line = "git tag -m init init" + log.debug("Executing '$cmd_line' in $path") + cmd = cmd_line.execute(envVars,path) + cmd.waitForProcessOutput(sout, serr) + + println sout + println serr + + + } + } } diff --git a/src/main/groovy/togit/migration/targets/git/context/GitTargetContext.groovy b/src/main/groovy/togit/migration/targets/git/context/GitTargetContext.groovy index b0a0af0..7ac02ac 100644 --- a/src/main/groovy/togit/migration/targets/git/context/GitTargetContext.groovy +++ b/src/main/groovy/togit/migration/targets/git/context/GitTargetContext.groovy @@ -75,4 +75,20 @@ class GitTargetContext extends TargetContext { target.options.remote = remoteUrl LOG.debug("Set remote url to $remoteUrl.") } + + /** + * Sets the Jira project key for git commit references + */ + void jiraProjectKey(String jiraProjectKey) { + log.debug("Set jiraProjectKey to $jiraProjectKey.") + target.options.jiraProjectKey = jiraProjectKey + } + + /** + * Sets the Jira project key for git commit references + */ + void jiraProjectKey(String jiraProjectKey) { + log.debug("Set jiraProjectKey to $jiraProjectKey.") + target.options.jiraProjectKey = jiraProjectKey + } } diff --git a/src/main/resources/examples/ccm2git-flatstructure-cust1.groovy b/src/main/resources/examples/ccm2git-flatstructure-cust1.groovy new file mode 100644 index 0000000..5dcec20 --- /dev/null +++ b/src/main/resources/examples/ccm2git-flatstructure-cust1.groovy @@ -0,0 +1,288 @@ +package examples + +/* vim: set syntax=groovy:set et:set tabstop=4: */ + +import org.slf4j.LoggerFactory + +final log = LoggerFactory.getLogger(this.class) + +def ccm_delimiter='~' + +def ccm_project +def ccm_revision +def ccm_name4part +def ccm_instance +if ( !start_project?.trim() || !start_project.contains(':') || !start_project.contains(ccm_delimiter) ) { + println "start_project not set correctly \n" + + "Provide the start_project=~:project:" + System.exit(1) +} else { + ccm_name4part = start_project.trim() + ccm_project = start_project.split(ccm_delimiter)[0] + ccm_revision = start_project.split(ccm_delimiter)[1].split(':')[0] + ccm_instance = start_project.split(ccm_delimiter)[1].split(':')[2] + + if ( ! ccm_project ) { + println "Could not extract ccm_project name from start_project" + System.exit(1) + } + if ( !ccm_revision || ccm_revision.contains(':') || ccm_revision.contains('~') ) { + println "ccm_revision contains ':' \n" + + "Provide the start_project=~:project:" + System.exit(1) + } + if ( !ccm_instance || ccm_instance.contains(':') || ccm_instance.contains('~') ) { + println "ccm_instance contains ':' or '~' \n" + + "Provide the start_project=~:project:" + System.exit(1) + } + if ( !ccm_name4part.contains(':') || !ccm_name4part.contains(ccm_delimiter) ) { + println "Provide the start_project=~:project:" + System.exit(1) + } +} + +def ccm_addr_cli +if ( !System.getenv("CCM_ADDR") ){ + println "CCM_ADDR system variable not set. Please start session prior to calling the ccm2git \n \ +( ex: ccm start -m -d /data/ccmdb/ -s http://: -q ) " + System.exit(1) +} else { + ccm_addr_cli = System.getenv("CCM_ADDR") +} + +def ccm_home_cli +if ( !System.getenv("CCM_HOME") ){ + println "CCM_HOME variable not set" + System.exit(1) +} else { + ccm_home_cli = System.getenv("CCM_HOME") +} +def system_path2 = System.getenv("PATH") + +def my_workspace +if ( !my_workspace_root ) { + my_workspace_root = "/data/Synergy/ccm2git-main" + my_workspace = my_workspace_root + "/" + ccm_project + "/" +} else { + my_workspace = my_workspace_root + "/" + ccm_project + "/" +} + +def git_server_path_this +if ( !git_server_path ){ + System.exit(1) +} else { + git_server_path_this = git_server_path +} + +def jira_project_key_this +if ( !jiraProjectKey ) { + println "Please set jiraProjectKey variable\n" + System.exit(1) +} else { + jiraProjectKeyThis = jiraProjectKey +} + +def my_workspace_file = new File(my_workspace) +if(!my_workspace_file.exists()) my_workspace_file.mkdirs() +my_workspace_file = new File(my_workspace + "/ccm_wa") +if(!my_workspace_file.exists()) my_workspace_file.mkdirs() + + +source('ccm') { + workspace "${my_workspace}/ccm_wa" + revision start_project + proj_instance ccm_instance + name4part ccm_name4part + ccm_addr ccm_addr_cli + ccm_home ccm_home_cli + system_path system_path2 + jiraProjectKey jiraProjectKeyThis +} + +target('git', repository_name) { + workspace "${my_workspace}/repo/" + ccm_project + user 'Claus Schneider(Eficode)' + email 'claus.schneider-ext@eficode.com' + remote "ssh://git@${git_server_path_this}/${ccm_project}.git" + longPaths true + ignore "" +} + +migrate { + filters { + filter { + criteria { + AlreadyConverted(target.workspace) + } + extractions { + baselineProperties(source.workspace, source.jiraProjectKey) + } + actions { + + // Scrub Git repository, so file deletions will also be committed + cmd 'git reset --hard -q $gitBaselineRevision_wstatus', target.workspace + + custom { + log.info "Removing files except .git folder in: $target.workspace" + new File(target.workspace).eachFile { file -> + if(!file.name.startsWith(".git")) { + if (!file.isDirectory()) { + println file.getName() + file.delete() + } else { + println file.getName() + file.deleteDir() + } + } + } + log.info "Remaining files except .git folder in: $target.workspace" + new File(target.workspace).eachFile { file -> + if(!file.name.startsWith(".git")) println file.getName() + } + } + + // Copy checked out into Git repository + copy("$source.workspace/code/\${gitSnapshotName}~\${gitSnapshotRevision}/\$gitSnapshotName", target.workspace) + + custom { + log.info "First level files in: $target.workspace" + new File(target.workspace).eachFile { file -> + if(!file.name.startsWith(".git")) println file.getName() + } + } + + // Remove all .gitignore, .gitmodules, .gitattributes except in root folder + cmd "bash git-remove-all-git-related-files-2plus-levels.sh " + target.workspace, System.getProperty("user.dir") + // Add everything and renormalize attributes + cmd 'git add -A --force .', target.workspace + cmd 'git add --renormalize -A --force .', target.workspace + + // Fill empty dirs with .gitignore for empty directories + cmd "bash git-fill-empty-dirs-with-gitignore.sh " + target.workspace, System.getProperty("user.dir") + // Add everything + cmd 'git add -A --force .', target.workspace + + // Update index to have executables on specific extensions + cmd "bash git-set-execute-bit-in-index-of-extensions.sh " + target.workspace, System.getProperty("user.dir") + + // Update index to have executables based on unix tool file reporting + cmd "bash git-set-execute-bit-in-index-of-unix-tool-file-executable.sh " + target.workspace, System.getProperty("user.dir") + + custom { project -> + def sout = new StringBuilder(), serr = new StringBuilder() + + new File(target.workspace + File.separator + ".." + File.separator + "commit_meta_data.txt").withWriter { out -> + project.baseline_commit_info.each { + out.println it + } + } + + def email_domain = '@eficode.com' + def envVars = System.getenv().collect { k, v -> "$k=$v" } + envVars.add('GIT_COMMITTER_DATE=' + project.snapshot_commiter_date) + envVars.add('GIT_AUTHOR_DATE=' + project.snapshot_commiter_date) + log.info("project.snapshotOwner: " + project.snapshotOwner) + if ( project.snapshotOwner != null ){ + envVars.add('GIT_AUTHOR_NAME=' + project.snapshotOwner ) + envVars.add('GIT_AUTHOR_EMAIL=' + project.snapshotOwner + email_domain) + } + def cmd_line = 'git commit --file ../commit_meta_data.txt' + log.info cmd_line.toString() + try { + def cmd = cmd_line.execute(envVars, new File(target.workspace)) + cmd.waitForProcessOutput(sout, serr) + def exitValue = cmd.exitValue() + log.info "Standard out: " + "\n" + sout.toString() + if (exitValue) { + if ( ! sout.contains('nothing to commit, working tree clean') ){ + log.error "Standard error:" + "'" + serr.toString() + "'" + log.error "Exit code: " + exitValue + throw new Exception(cmd_line + ": gave exit code: $exitValue") + } else { + log.info "Nothing commit - skip, but still tag" + } + if (serr.toString().readLines().size() > 0) { + log.error "Standard error: " + "'" + serr.toString() + "'" + log.error "Exit code: " + exitValue + throw new Exception(cmd_line + ": standard error contains text lines: " + serr.toString().readLines().size()) + } + } + if (serr.toString().readLines().size() > 0) { + log.info (cmd_line + ": standard error contains text lines: " + serr.toString().readLines().size()) + log.info "Standard error:" + "'" + serr.toString() + "'" + } + } catch (Exception e) { + log.error('An error occurred during the git commit..') + log.error(e.toString()) + throw e + } + + } + + // Reset to test that git return to workspace is identical except the .git* files that are manipulated (removed Synergy snapshot .git files and added .gitignore to empty dirs + custom { + log.info "Removing files except .git folder in: $target.workspace" + new File(target.workspace).eachFile { file -> + if(!file.name.startsWith(".git")) { + if (!file.isDirectory()) { + println file.getName() + file.delete() + } else { + println file.getName() + file.deleteDir() + } + } + } + } + cmd 'git reset --hard -q HEAD', target.workspace + cmd 'diff -r -q -x ".gitignore" -x ".gitattributes" -x ".gitmodules" -x ".git" . ' + source.workspace + '/code/${gitSnapshotName}~${gitSnapshotRevision}/${gitSnapshotName}', target.workspace + + // The file for tag info is generated during MetaDataExtraction + custom { project -> + new File(target.workspace + File.separator + ".." + File.separator + "tag_meta_data.txt").withWriter { out -> + project.baseline_tag_info.each { + out.println it + } + } + } + custom { project -> + def sout = new StringBuilder(), serr = new StringBuilder() + def cmd_line = "git tag -F ../tag_meta_data.txt " + project.gitSnapshotRevision + "_" + project.snapshot_status + log.info cmd_line + + def email_domain = '@eficode.com' + def envVars = System.getenv().collect { k, v -> "$k=$v" } + envVars.add('GIT_COMMITTER_DATE=' + project.snapshot_commiter_date) + envVars.add('GIT_AUTHOR_DATE=' + project.snapshot_commiter_date) + log.info("project.snapshotOwner: " + project.snapshotOwner) + if ( project.snapshotOwner != null ){ + envVars.add('GIT_COMMITTER_NAME=' + project.snapshotOwner ) + envVars.add('GIT_COMMITTER_EMAIL=' + project.snapshotOwner + email_domain) + } + def cmd = cmd_line.execute(envVars,new File(target.workspace)) + cmd.waitForProcessOutput(sout, serr) + def exitValue = cmd.exitValue() + log.info "Standard out:" + println "'" + sout.toString() + "'" + if ( exitValue ){ + log.info "Standard error:" + println "'" + serr.toString() + "'" + log.info "Exit code: " + exitValue + throw new Exception(cmd_line + ": gave exit code: $exitValue" ) + } + if ( serr.toString().readLines().size() > 0 ){ + log.info "Standard error:" + println "'" + serr.toString() + "'" + log.info "Exit code: " + exitValue + throw new Exception(cmd_line + ": standard error contains text lines: " + serr.toString().readLines().size() ) + } + } + + cmd 'du -sBM .git > ../${gitSnapshotName}~${gitSnapshotRevision}@git_size.txt', target.workspace + cmd 'cat ../${gitSnapshotName}~${gitSnapshotRevision}@git_size.txt', target.workspace + + } + } + } +} diff --git a/src/main/resources/examples/ccm2git-flatstructure-lfs.groovy b/src/main/resources/examples/ccm2git-flatstructure-lfs.groovy new file mode 100644 index 0000000..212f90f --- /dev/null +++ b/src/main/resources/examples/ccm2git-flatstructure-lfs.groovy @@ -0,0 +1,265 @@ +package examples + +/* vim: set syntax=groovy:set et:set tabstop=4: */ + +import org.slf4j.LoggerFactory + +final log = LoggerFactory.getLogger(this.class) + +def ccm_delimiter='~' + +def ccm_project +def ccm_revision +def ccm_name4part +def ccm_instance +if ( !start_project?.trim() || !start_project.contains(':') || !start_project.contains(ccm_delimiter) ) { + println "start_project not set correctly \n" + + "Provide the start_project=" + ccm_delimiter + ":project:" + System.exit(1) +} else { + ccm_name4part = start_project.trim() + ccm_project = start_project.split(ccm_delimiter)[0] + ccm_revision = start_project.split(ccm_delimiter)[1].split(':')[0] + ccm_instance = start_project.split(ccm_delimiter)[1].split(':')[2] + + if ( ! ccm_project ) { + println "Could not extract ccm_project name from start_project" + System.exit(1) + } + if ( !ccm_revision || ccm_revision.contains(':') || ccm_revision.contains('~') ) { + println "ccm_revision contains ':' \n" + + "Provide the start_project=" + ccm_delimiter + ":project:" + System.exit(1) + } + if ( !ccm_instance || ccm_instance.contains(':') || ccm_instance.contains('~') ) { + println "ccm_instance contains ':' or '~' \n" + + "Provide the start_project=" + ccm_delimiter + ":project:" + System.exit(1) + } + if ( !ccm_name4part.contains(':') || !ccm_name4part.contains(ccm_delimiter) ) { + println "Provide the start_project=" + ccm_delimiter + ":project:" + System.exit(1) + } +} + +def ccm_addr_cli +if ( !System.getenv("CCM_ADDR") ){ + println "CCM_ADDR system variable not set. Please start session prior to calling the ccm2git \n \ +( ex: ccm start -m -d /data/ccmdb/ -s http://: -q ) " + System.exit(1) +} else { + ccm_addr_cli = System.getenv("CCM_ADDR") +} + +def ccm_home_cli +if ( !System.getenv("CCM_HOME") ){ + println "CCM_HOME variable not set" + System.exit(1) +} else { + ccm_home_cli = System.getenv("CCM_HOME") +} +def system_path2 = System.getenv("PATH") + +def my_workspace +if ( !my_workspace_root ) { + my_workspace_root = "/data/Synergy/ccm2git-main" + my_workspace = my_workspace_root + "/" + ccm_project + "/" +} else { + my_workspace = my_workspace_root + "/" + ccm_project + "/" +} + +def git_server_path_this +if ( !git_server_path ){ + System.exit(1) +} else { + git_server_path_this = git_server_path +} + +def jira_project_key_this +if ( !jiraProjectKey ) { + println "Please set jiraProjectKey variable\n" + System.exit(1) +} else { + jiraProjectKeyThis = jiraProjectKey +} + +def my_workspace_file = new File(my_workspace) +if(!my_workspace_file.exists()) my_workspace_file.mkdirs() +my_workspace_file = new File(my_workspace + "/ccm_wa") +if(!my_workspace_file.exists()) my_workspace_file.mkdirs() + + +source('ccm') { + workspace "${my_workspace}/ccm_wa" + revision start_project + proj_instance ccm_instance + name4part ccm_name4part + ccm_addr ccm_addr_cli + ccm_home ccm_home_cli + system_path system_path2 + jiraProjectKey jiraProjectKeyThis +} + +target('git', repository_name) { + workspace "${my_workspace}/repo/" + ccm_project + user 'Claus Schneider(Eficode)' + email 'claus.schneider.ext@eficode.com' + remote "ssh://git@${git_server_path_this}/${ccm_project}.git" + longPaths true + ignore "" + lfs '*.bundle', '*.pima', '*.bz2', '*.exe', '*.jar', '*.deb', '*.zip', '*.tgz', '*.xz', '*.gz' +} + +migrate { + filters { + filter { + criteria { + AlreadyConverted(target.workspace) + } + extractions { + baselineProperties(source.workspace, source.jiraProjectKey) + } + actions { + + // Scrub Git repository, so file deletions will also be committed + cmd 'git reset --hard -q $baselineRevision_wstatus', target.workspace + + custom { + log.info "Removing files except .git folder in: $target.workspace" + new File(target.workspace).eachFile { file -> + if(!file.name.startsWith(".git")) { + if (!file.isDirectory()) { + println file.getName() + file.delete() + } else { + println file.getName() + file.deleteDir() + } + } + } + log.info "Remaining files except .git folder in: $target.workspace" + new File(target.workspace).eachFile { file -> + if(!file.name.startsWith(".git")) println file.getName() + } + } + + // Copy checked out into Git repository + copy("$source.workspace/code/\${snapshotName}-\${snapshotRevision}/\$snapshotName", target.workspace) + + custom { + log.info "First level files in: $target.workspace" + new File(target.workspace).eachFile { file -> + if(!file.name.startsWith(".git")) println file.getName() + } + } + + // Fill empty dirs with .gitignore for empty directories + cmd "bash git-fill-empty-dirs-with-gitignore.sh " + target.workspace, System.getProperty("user.dir") + + // Add everything + cmd 'git add -A --force .', target.workspace + + // Update index to have executables on specific extensions + cmd "bash git-set-execute-bit-in-index-of-extensions.sh " + target.workspace, System.getProperty("user.dir") + + // Update index to have executables based on unix tool file reporting + cmd "bash git-set-execute-bit-in-index-of-unix-tool-file-executable.sh " + target.workspace, System.getProperty("user.dir") + + custom { project -> + def sout = new StringBuilder(), serr = new StringBuilder() + + new File(target.workspace + File.separator + ".." + File.separator + "commit_meta_data.txt").withWriter { out -> + project.baseline_commit_info.each { + out.println it + } + } + + def email_domain = '@eficode.com' + def envVars = System.getenv().collect { k, v -> "$k=$v" } + envVars.add('GIT_COMMITTER_DATE=' + project.snapshot_commiter_date) + envVars.add('GIT_AUTHOR_DATE=' + project.snapshot_commiter_date) + log.info("project.snapshotOwner: " + project.snapshotOwner) + if ( project.snapshotOwner != null ){ + envVars.add('GIT_AUTHOR_NAME=' + project.snapshotOwner ) + envVars.add('GIT_AUTHOR_EMAIL=' + project.snapshotOwner + email_domain) + } + def cmd_line = 'git commit --file ../commit_meta_data.txt' + log.info cmd_line.toString() + try { + def cmd = cmd_line.execute(envVars, new File(target.workspace)) + cmd.waitForProcessOutput(sout, serr) + def exitValue = cmd.exitValue() + log.info "Standard out: " + "\n" + sout.toString() + if (exitValue) { + if ( ! sout.contains('nothing to commit, working tree clean') ){ + log.error "Standard error:" + "'" + serr.toString() + "'" + log.error "Exit code: " + exitValue + throw new Exception(cmd_line + ": gave exit code: $exitValue") + } else { + log.info "Nothing commit - skip, but still tag" + } + if (serr.toString().readLines().size() > 0) { + log.error "Standard error: " + "'" + serr.toString() + "'" + log.error "Exit code: " + exitValue + throw new Exception(cmd_line + ": standard error contains text lines: " + serr.toString().readLines().size()) + } + } + if (serr.toString().readLines().size() > 0) { + log.info (cmd_line + ": standard error contains text lines: " + serr.toString().readLines().size()) + log.info "Standard error:" + "'" + serr.toString() + "'" + } + } catch (Exception e) { + log.error('An error occurred during the git commit..') + log.error(e.toString()) + throw e + } + + } + + // The file for tag info is generated during MetaDataExtraction + custom { project -> + new File(target.workspace + File.separator + ".." + File.separator + "tag_meta_data.txt").withWriter { out -> + project.baseline_tag_info.each { + out.println it + } + } + } + custom { project -> + def sout = new StringBuilder(), serr = new StringBuilder() + def cmd_line = "git tag -F ../tag_meta_data.txt " + project.snapshotRevision + "_" + project.snapshot_status + log.info cmd_line + + def email_domain = '@eficode.com' + def envVars = System.getenv().collect { k, v -> "$k=$v" } + envVars.add('GIT_COMMITTER_DATE=' + project.snapshot_commiter_date) + envVars.add('GIT_AUTHOR_DATE=' + project.snapshot_commiter_date) + log.info("project.snapshotOwner: " + project.snapshotOwner) + if ( project.snapshotOwner != null ){ + envVars.add('GIT_COMMITTER_NAME=' + project.snapshotOwner ) + envVars.add('GIT_COMMITTER_EMAIL=' + project.snapshotOwner + email_domain) + } + def cmd = cmd_line.execute(envVars,new File(target.workspace)) + cmd.waitForProcessOutput(sout, serr) + def exitValue = cmd.exitValue() + log.info "Standard out:" + println "'" + sout.toString() + "'" + if ( exitValue ){ + log.info "Standard error:" + println "'" + serr.toString() + "'" + log.info "Exit code: " + exitValue + throw new Exception(cmd_line + ": gave exit code: $exitValue" ) + } + if ( serr.toString().readLines().size() > 0 ){ + log.info "Standard error:" + println "'" + serr.toString() + "'" + log.info "Exit code: " + exitValue + throw new Exception(cmd_line + ": standard error contains text lines: " + serr.toString().readLines().size() ) + } + } + + cmd 'du -sBM .git > ../${snapshotName}-${snapshotRevision}@git_size.txt', target.workspace + cmd 'cat ../${snapshotName}-${snapshotRevision}@git_size.txt', target.workspace + } + } + } +} diff --git a/src/main/resources/examples/ccm2git-flatstructure.groovy b/src/main/resources/examples/ccm2git-flatstructure.groovy new file mode 100644 index 0000000..ad0d953 --- /dev/null +++ b/src/main/resources/examples/ccm2git-flatstructure.groovy @@ -0,0 +1,355 @@ +package examples + +/* vim: set syntax=groovy:set et:set tabstop=4: */ + +import org.slf4j.LoggerFactory + +final log = LoggerFactory.getLogger(this.class) + +if ( !System.getenv("git_user_name") ){ + println "ERROR: git_user_name env variable must be set. It is used for the committer and init commit author." + System.exit(1) +} else { + println "git_user_name: " + System.getenv("git_user_name") +} +if ( !System.getenv("git_user_email") ){ + println "ERROR: git_user_email env variable must be set. It is used for the committer and init commit author" + System.exit(1) +} else { + println "git_user_email: " + System.getenv("git_user_email") +} +if ( !System.getenv("git_email_domain") ){ + println "ERROR: git_email_domain env variable must be set. It is used for the author domain ala 'eficode.com'. The username part of email is retrieved from CM/Synergy" + System.exit(1) +} else { + println "git_email_domain: " + System.getenv("git_email_domain") +} + +if ( !System.getenv("git_user_name") ){ + println "ERROR: git_user_name env variable must be set. It is used for the committer and init commit author." + System.exit(1) +} else { + println "git_user_name: " + System.getenv("git_user_name") +} +if ( !System.getenv("git_user_email") ){ + println "ERROR: git_user_email env variable must be set. It is used for the committer and init commit author" + System.exit(1) +} else { + println "git_user_email: " + System.getenv("git_user_email") +} +if ( !System.getenv("git_email_domain") ){ + println "ERROR: git_email_domain env variable must be set. It is used for the author domain ala 'eficode.com'. The username part of email is retrieved from CM/Synergy" + System.exit(1) +} else { + println "git_email_domain: " + System.getenv("git_email_domain") +} + +if ( !System.getenv("ccm_delim") ){ + println "ccm_delim variable not set" + System.exit(1) +} else { + println "ccm_delim: " + System.getenv("ccm_delim") + ccm_delimiter = System.getenv("ccm_delim") +} + + +def ccm_project +def ccm_revision +def ccm_name4part +def ccm_instance +if ( !start_project?.trim() || !start_project.contains(':') || !start_project.contains(ccm_delimiter) ) { + println "start_project not set correctly \n" + + "Provide the start_project=" + ccm_delimiter + ":project:" + "Provide the start_project=" + ccm_delimiter + ":project:" + System.exit(1) +} else { + ccm_name4part = start_project.trim() + ccm_project = start_project.split(ccm_delimiter)[0] + ccm_revision = start_project.split(ccm_delimiter)[1].split(':')[0] + ccm_instance = start_project.split(ccm_delimiter)[1].split(':')[2] + + if ( ! ccm_project ) { + println "Could not extract ccm_project name from start_project" + System.exit(1) + } + if ( !ccm_revision || ccm_revision.contains(':') || ccm_revision.contains('~') ) { + println "ccm_revision contains ':' \n" + + "Provide the start_project=" + ccm_delimiter + ":project:" + "Provide the start_project=" + ccm_delimiter + ":project:" + System.exit(1) + } + if ( !ccm_instance || ccm_instance.contains(':') || ccm_instance.contains('~') ) { + println "ccm_instance contains ':' or '~' \n" + + "Provide the start_project=" + ccm_delimiter + ":project:" + "Provide the start_project=" + ccm_delimiter + ":project:" + System.exit(1) + } + if ( !ccm_name4part.contains(':') || !ccm_name4part.contains(ccm_delimiter) ) { + println "Provide the start_project=" + ccm_delimiter + ":project:" + println "Provide the start_project=" + ccm_delimiter + ":project:" + System.exit(1) + } +} + +def ccm_addr_cli +if ( !System.getenv("CCM_ADDR") ){ + println "CCM_ADDR system variable not set. Please start session prior to calling the ccm2git \n \ +( ex: ccm start -m -d /data/ccmdb/ -s http://: -q ) " + System.exit(1) +} else { + ccm_addr_cli = System.getenv("CCM_ADDR") +} + +def ccm_home_cli +if ( !System.getenv("CCM_HOME") ){ + println "CCM_HOME variable not set" + System.exit(1) +} else { + ccm_home_cli = System.getenv("CCM_HOME") +} +def system_path2 = System.getenv("PATH") +<<<<<<< HEAD +======= + + + +>>>>>>> origin/ccm2git-4part-dynamic-db-delimiter +def my_workspace +if ( !my_workspace_root ) { + my_workspace_root = "/data/Synergy/ccm2git-main" + my_workspace = my_workspace_root + "/" + ccm_project + "/" +} else { + my_workspace = my_workspace_root + "/" + ccm_project + "/" +} + +def git_server_path_this +if ( !git_server_path ){ + System.exit(1) +} else { + git_server_path_this = git_server_path +} + + + +def jira_project_key_this +if ( !jiraProjectKey ) { + println "Please set jiraProjectKey variable\n" + System.exit(1) +} else { + jiraProjectKeyThis = jiraProjectKey +} + +def my_workspace_file = new File(my_workspace) +if(!my_workspace_file.exists()) my_workspace_file.mkdirs() +my_workspace_file = new File(my_workspace + "/ccm_wa") +if(!my_workspace_file.exists()) my_workspace_file.mkdirs() + + +source('ccm') { + workspace "${my_workspace}/ccm_wa" + revision start_project + proj_instance ccm_instance + name4part ccm_name4part + ccm_addr ccm_addr_cli + ccm_home ccm_home_cli + system_path system_path2 + jiraProjectKey jiraProjectKeyThis +} + +target('git', repository_name) { + workspace "${my_workspace}/repo/" + ccm_project + user System.getenv("git_user_name") + email System.getenv("git_user_email") + remote "ssh://git@${git_server_path_this}/${ccm_project}.git" + longPaths true + ignore "" +} + +migrate { + filters { + filter { + criteria { + AlreadyConverted(target.workspace) + } + + extractions { + baselineProperties(source.workspace, source.jiraProjectKey) + } + actions { + + // Scrub Git repository, so file deletions will also be committed + // THIS SHOULD BE GENERIC AND BE IN BASE GIT SOURCE + cmd 'git reset --hard -q $gitBaselineRevision_wstatus', target.workspace + custom { + log.info "Removing files except .git folder in: $target.workspace" + new File(target.workspace).eachFile { file -> + if(!file.name.startsWith(".git")) { + if (!file.isDirectory()) { + println file.getName() + file.delete() + } else { + println file.getName() + file.deleteDir() + } + } + } + log.info "Remaining files except .git folder in: $target.workspace" + new File(target.workspace).eachFile { file -> + if(!file.name.startsWith(".git")) println file.getName() + } + } + // End scrub + + // Copy checked out into Git repository + copy("$source.workspace/code/\${gitSnapshotName}-\${gitSnapshotRevision}/\$gitSnapshotName", target.workspace) + + // DEBUG INFO + custom { + log.info "First level files in: $target.workspace" + new File(target.workspace).eachFile { file -> + if(!file.name.startsWith(".git")) println file.getName() + } + } + + // Remove all .gitignore, .gitmodules, .gitattributes except in root folder + cmd "bash git-remove-all-git-related-files-2plus-levels.sh " + target.workspace, System.getProperty("user.dir") + // Add everything and renormalize attributes + cmd 'git add -A --force .', target.workspace + cmd 'git add --renormalize -A --force .', target.workspace + + // Fill empty dirs with .gitignore for empty directories + cmd "bash git-fill-empty-dirs-with-gitignore.sh " + target.workspace, System.getProperty("user.dir") + // Add everything + cmd 'git add -A --force .', target.workspace + + // Update index to have executables on specific extensions + cmd "bash git-set-execute-bit-in-index-of-extensions.sh " + target.workspace, System.getProperty("user.dir") + + // Update index to have executables based on unix tool file reporting + cmd "bash git-set-execute-bit-in-index-of-unix-tool-file-executable.sh " + target.workspace, System.getProperty("user.dir") + + custom { project -> + def sout = new StringBuilder(), serr = new StringBuilder() + + new File(target.workspace + File.separator + ".." + File.separator + "commit_meta_data.txt").withWriter { out -> + project.baseline_commit_info.each { + out.println it + } + } + if ( !System.getenv("git_email_domain") ){ + println "ERROR: git_email_domain env variable must be set. It is used for the author domain ala 'eficode.com'. The username part of email is retrieved from CM/Synergy" + System.exit(1) + } + def email_domain = '@'+System.getenv("git_email_domain") + log.info("email_domain: " + email_domain ) + + def envVars = System.getenv().collect { k, v -> "$k=$v" } + envVars.add('GIT_COMMITTER_DATE=' + project.snapshot_commiter_date) + envVars.add('GIT_AUTHOR_DATE=' + project.snapshot_commiter_date) + log.info("project.snapshotOwner: " + project.snapshotOwner) + if (project.snapshotOwner){ + envVars.add('GIT_AUTHOR_NAME=' + project.snapshotOwner ) + envVars.add('GIT_AUTHOR_EMAIL=' + project.snapshotOwner + email_domain) + } + def cmd_line = 'git commit --file ../commit_meta_data.txt' + log.info cmd_line.toString() + try { + def cmd = cmd_line.execute(envVars, new File(target.workspace)) + cmd.waitForProcessOutput(sout, serr) + def exitValue = cmd.exitValue() + log.info "Standard out: " + "\n" + sout.toString() + if (exitValue) { + if ( ! sout.contains('nothing to commit, working tree clean') ){ + log.error "Standard error:" + "'" + serr.toString() + "'" + log.error "Exit code: " + exitValue + throw new Exception(cmd_line + ": gave exit code: $exitValue") + } else { + log.info "Nothing commit - skip, but still tag" + } + if (serr.toString().readLines().size() > 0) { + log.error "Standard error: " + "'" + serr.toString() + "'" + log.error "Exit code: " + exitValue + throw new Exception(cmd_line + ": standard error contains text lines: " + serr.toString().readLines().size()) + } + } + if (serr.toString().readLines().size() > 0) { + log.info (cmd_line + ": standard error contains text lines: " + serr.toString().readLines().size()) + log.info "Standard error:" + "'" + serr.toString() + "'" + } + } catch (Exception e) { + log.error('An error occurred during the git commit..') + log.error(e.toString()) + throw e + } + + } + + // Reset to test that git return to workspace is identical except the .git* files that are manipulated (removed Synergy snapshot .git files and added .gitignore to empty dirs + custom { + log.info "Removing files except .git folder in: $target.workspace" + new File(target.workspace).eachFile { file -> + if(!file.name.startsWith(".git")) { + if (!file.isDirectory()) { + println file.getName() + file.delete() + } else { + println file.getName() + file.deleteDir() + } + } + } + } + cmd 'git reset --hard -q HEAD', target.workspace + cmd 'diff -r -q -x ".gitignore" -x ".gitattributes" -x ".gitmodules" -x ".git" . ' + source.workspace + '/code/${gitSnapshotName}-${gitSnapshotRevision}/${gitSnapshotName}', target.workspace + cmd 'diff -r -q -x ".gitignore" -x ".gitattributes" -x ".gitmodules" -x ".git" . ' + source.workspace + '/code/${gitSnapshotName}-${gitSnapshotRevision}/${gitSnapshotName}', target.workspace + + // The file for tag info is generated during MetaDataExtraction + custom { project -> + new File(target.workspace + File.separator + ".." + File.separator + "tag_meta_data.txt").withWriter { out -> + project.baseline_tag_info.each { + out.println it + } + } + } + custom { project -> + def sout = new StringBuilder(), serr = new StringBuilder() + def cmd_line = "git tag -F ../tag_meta_data.txt " + project.gitSnapshotRevision + "_" + project.snapshot_status + log.info cmd_line + + def email_domain = '@safrangroup.com' + def email_domain = '@safrangroup.com' + def envVars = System.getenv().collect { k, v -> "$k=$v" } + envVars.add('GIT_COMMITTER_DATE=' + project.snapshot_commiter_date) + envVars.add('GIT_AUTHOR_DATE=' + project.snapshot_commiter_date) + log.info("project.snapshotOwner: " + project.snapshotOwner) + if ( project.snapshotOwner != null ){ + envVars.add('GIT_COMMITTER_NAME=' + project.snapshotOwner ) + envVars.add('GIT_COMMITTER_EMAIL=' + project.snapshotOwner + email_domain) + } + def cmd = cmd_line.execute(envVars,new File(target.workspace)) + cmd.waitForProcessOutput(sout, serr) + def exitValue = cmd.exitValue() + log.info "Standard out:" + println "'" + sout.toString() + "'" + if ( exitValue ){ + log.info "Standard error:" + println "'" + serr.toString() + "'" + log.info "Exit code: " + exitValue + throw new Exception(cmd_line + ": gave exit code: $exitValue" ) + } + if ( serr.toString().readLines().size() > 0 ){ + log.info "Standard error:" + println "'" + serr.toString() + "'" + log.info "Exit code: " + exitValue + throw new Exception(cmd_line + ": standard error contains text lines: " + serr.toString().readLines().size() ) + } + } + // Show the size of the .git directory for information purposes + // Useful to see if something is wrong and the .git directory is growing too large + cmd 'du -sBM .git > ../${gitSnapshotName}-${gitSnapshotRevision}@git_size.txt', target.workspace + cmd 'cat ../${gitSnapshotName}-${gitSnapshotRevision}@git_size.txt', target.workspace + + } + } + } +} diff --git a/src/main/resources/toGit/migration/sources/ccm/.gitignore b/src/main/resources/toGit/migration/sources/ccm/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/src/main/resources/toGit/migration/sources/ccm/_ccm-functions-test.sh b/src/main/resources/toGit/migration/sources/ccm/_ccm-functions-test.sh new file mode 100644 index 0000000..47db9e8 --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/_ccm-functions-test.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Load functions +source ./_ccm-functions.sh || source ${BASH_SOURCE%/*}/_ccm-functions.sh + +[[ "${debug:-}" == "true" ]] && set -x + + +set -euo pipefail + +ccm_project_name=" " +expected_result="-" +result="" +printf "%-8s: %-80s : %-75s " "test" "byref_translate_from_ccm_name2git_repo" "${ccm_project_name} -> $expected_result" +byref_translate_from_ccm_name2git_repo "$ccm_project_name" result +[[ "$result" == "${expected_result}" ]] && { printf "%10s\n" "SUCCESS" ; }|| { printf " FAILED: ${expected_result} != $result\n" ;} + +ccm_project_name="-" +expected_result="?" +result="" +printf "%-8s: %-80s : %-75s " "test" "byref_translate_from_git_repo2ccm_name_query" "${ccm_project_name} -> $expected_result" +byref_translate_from_git_repo2ccm_name_query $ccm_project_name result +[[ "$result" == "${expected_result}" ]] && { printf "%10s\n" "SUCCESS" ; }|| { printf " FAILED: ${expected_result} != $result\n" ;} + +ccm_query_name_instance_string="?" +ccm_query_instance="1" +expected_result=" " +result="" +printf "%-8s: %-80s : %-75s " "test" "byref_translate_from_ccm_name_instance_query2ccm_name" "${ccm_query_name_instance_string} -> $expected_result" +byref_translate_from_ccm_name_instance_query2ccm_name $ccm_query_name_instance_string $ccm_query_instance result +[[ "$result" == "${expected_result}" ]] && { printf "%10s\n" "SUCCESS" ; }|| { printf " FAILED: ${expected_result} != $result\n" ;} + +git_repo_name="-" +ccm_query_instance="1" +expected_result=" " +result="" +printf "%-8s: %-80s : %-75s " "test" "byref_translate_from_git_repo2ccm_name" "${ccm_project_name} -> $expected_result" +byref_translate_from_git_repo2ccm_name $git_repo_name $ccm_query_instance result +[[ "$result" == "${expected_result}" ]] && { printf "%10s\n" "SUCCESS" ; }|| { printf " FAILED: ${expected_result} != $result\n" ;} + +git_repo_4part="-~<:project:1" +expected_result=" ~:project:1" +result="" +printf "%-8s: %-80s : %-75s " "test" "byref_translate_from_git_repo_4part2ccm_4part" "${git_repo_4part} -> $expected_result" +byref_translate_from_git_repo_4part2ccm_4part "$git_repo_4part" result +[[ "$result" == "${expected_result}" ]] && { printf "%10s\n" "SUCCESS" ; }|| { printf " FAILED: ${expected_result} != $result\n" ;} + +ccm_4part="~ :project:1" +expected_result="~-:project:1" +result="" +printf "%-8s: %-80s : %-75s " "test" "byref_translate_from_ccm_4part2git_repo_4part" "${ccm_4part} -> $expected_result" +byref_translate_from_ccm_4part2git_repo_4part "${ccm_4part}" result +[[ "$result" == "${expected_result}" ]] && { printf "%10s\n" "SUCCESS" ; }|| { printf " FAILED: ${expected_result} != $result\n" ;} + diff --git a/src/main/resources/toGit/migration/sources/ccm/_ccm-functions.sh b/src/main/resources/toGit/migration/sources/ccm/_ccm-functions.sh new file mode 100644 index 0000000..67f4c43 --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/_ccm-functions.sh @@ -0,0 +1,280 @@ +#!/usr/bin/env bash + +export regex_ccm4part="^(.+)${ccm_delim}(.+):(.+):(.+)$" + +function byref_translate_from_ccm_name2git_repo() { + if [[ -z ${1} ]]; then + echo "${FUNCNAME[0]}: Parameter 1 - by value - cannot be empty" && exit 1 + else + local _fromString=${1} + fi + if [[ -z ${2} ]]; then + echo "${FUNCNAME[0]}: Parameter 2 - as ref - cannot be empty" && exit 1 + else + local -n _toString=${2} + fi + _toString="${_fromString}" + _toString="${_toString//ener Tam/ener-Tam}" + _toString="${_toString// /-}" + _toString="${_toString//&/-}" + _toString="${_toString//#/-}" + _toString="${_toString//(/-}" + _toString="${_toString//)/-}" + _toString="${_toString//ü/ue}" + _toString="${_toString//ä/ae}" + _toString="${_toString//æ/ae}" + _toString="${_toString//å/aa}" + _toString="${_toString//ö/oe}" + _toString="${_toString//ø/oe}" + _toString="${_toString//,/-}" +} + +function byref_translate_from_git_repo2ccm_name_query() { + if [[ -z ${1} ]]; then + echo "${FUNCNAME[0]}: Parameter 1 - by value - cannot be empty" && exit 1 + else + local _fromString=${1} + fi + if [[ -z ${2} ]]; then + echo "${FUNCNAME[0]}: Parameter 2 - by ref - cannot be empty" && exit 1 + else + local -n _toString=${2} + fi + _toString="${_fromString}" + _toString="${_toString//ener-Tam/ener Tam}" + _toString="${_toString//-/?}" + _toString="${_toString//ue/??}" + _toString="${_toString//ae/??}" + _toString="${_toString//aa/??}" + _toString="${_toString//oe/??}" +} + +function byref_translate_from_ccm_version2git_tag() { +# https://stackoverflow.com/questions/26382234/what-names-are-valid-git-tags +#Disallowed characters: '~', '^', ':', '', '?', '[', '*' +#Disallowed sequences ".." and "@{" +#Disallowed Also tag cannot contain " " (whitespace) + if [[ -z ${1} ]]; then + echo "${FUNCNAME[0]}: Parameter 1 - by value - cannot be empty" && exit 1 + else + local _fromString=${1} + fi + if [[ -z ${2} ]]; then + echo "${FUNCNAME[0]}: Parameter 2 - as ref - cannot be empty" && exit 1 + else + local -n _toString=${2} + fi + _toString="${_fromString// /-}" + _toString="${_toString//&/-}" + _toString="${_toString//#/-}" + _toString="${_toString//(/-}" + _toString="${_toString//)/-}" +} + +function byref_translate_from_git_tag2ccm_version_query() { + if [[ -z ${1} ]]; then + echo "${FUNCNAME[0]}: Parameter 1 - by value - cannot be empty" && exit 1 + else + local _fromString=${1} + fi + if [[ -z ${2} ]]; then + echo "${FUNCNAME[0]}: Parameter 2 - by ref - cannot be empty" && exit 1 + else + local -n _toString=${2} + fi + _toString="${_fromString//-/?}" +} + + +function byref_translate_from_ccm_name_instance_query2ccm_name() { + if [[ -z ${1} ]]; then + echo "${FUNCNAME[0]}: Parameter 1 - by value - cannot be empty" && exit 1 + else + local _fromString=${1} + fi + if [[ -z ${2} ]]; then + echo "${FUNCNAME[0]}: Parameter 2 - by value - cannot be empty" && exit 1 + else + local _instance=${2} + fi + if [[ -z ${3} ]]; then + echo "${FUNCNAME[0]}: Parameter 3 - as ref - cannot be empty" && exit 1 + else + local -n _toString=${3} + fi + + local _query_string="name match '$_fromString' and type='project' and instance='${_instance}'" + local _found_project_name_instances=$(ccm query "${_query_string}" -u -f "%name:%instance" | /usr/bin/sort -u | wc -l) + if [[ _found_project_name_instances -eq 0 ]]; then + echo "ERROR: I found no projects with similar ? query name output gave foo and boo " + echo "$_query_string" + return 1 + fi + if [[ _found_project_name_instances -gt 1 ]]; then + echo "ERROR: I found two or more projects with similar ? query name output -oo gave foo and boo" + ccm query "${_query_string}" -u -f "%name:%instance" + return 1 + fi + _toString=$(ccm query "${_query_string}" -u -f "%name" | /usr/bin/sort -u ) +} + +function byref_translate_from_git_repo2ccm_name() { + if [[ -z ${1:-} ]]; then + echo "${FUNCNAME[0]}: Parameter 1 - by value - cannot be empty" && exit 1 + else + local _fromString=${1} + fi + if [[ -z ${2:-} ]]; then + echo "${FUNCNAME[0]}: Parameter 2 - by value - cannot be empty" && exit 1 + else + local _instance=${2} + fi + if [[ -z ${3:-} ]]; then + echo "${FUNCNAME[0]}: Parameter 3 - by ref - cannot be empty" && exit 1 + else + local -n _toString=${3} + fi + + local _git_repo_name=$_fromString + local _ccm_instance=$_instance + local _ccm_query_name="" + byref_translate_from_git_repo2ccm_name_query "${_git_repo_name}" _ccm_query_name + local _query_result="" + byref_translate_from_ccm_name_instance_query2ccm_name "${_ccm_query_name}" "${_instance}" _query_result + _toString=$_query_result +} + +function byref_translate_from_git_repo_4part2ccm_4part() { + if [[ -z ${1} ]]; then + echo "${FUNCNAME[0]}: Parameter 1 - by value - cannot be empty" && exit 1 + else + local _fromString=${1} + fi + if [[ -z ${2} ]]; then + echo "${FUNCNAME[0]}: Parameter 2 - by ref - cannot be empty" && exit 1 + else + local -n _toString=${2} + fi + + [[ "${_fromString:-}" =~ ${regex_ccm4part} ]] || { + echo "4part does not comply" + return 1 + } + local _name=${BASH_REMATCH[1]} + local _version=${BASH_REMATCH[2]} + local _type=${BASH_REMATCH[3]} + local _instance=${BASH_REMATCH[4]} + + local _ccm_query_name="" + byref_translate_from_git_repo2ccm_name_query "${_name}" _ccm_query_name + + local _ccm_query_version="" + byref_translate_from_git_tag2ccm_version_query "${_version}" _ccm_query_version + + local _ccm_query_4name="$_ccm_query_name-$_ccm_query_version:$_type:$_instance" + + local _query_result="" + byref_translate_from_ccm_4part_query2ccm_4part "${_ccm_query_4name}" _query_result + _toString=$_query_result +} + +function byref_translate_from_ccm_4part2git_repo_4part() { + if [[ -z ${1} ]]; then + echo "${FUNCNAME[0]}: Parameter 1 - by value - cannot be empty" && exit 1 + else + local _fromString=${1} + fi + if [[ -z ${2} ]]; then + echo "${FUNCNAME[0]}: Parameter 2 - by ref - cannot be empty" && exit 1 + else + local -n _toString=${2} + fi + + [[ "${_fromString:-}" =~ ${regex_ccm4part} ]] || { + echo "4part does not comply" + return 1 + } + local _name=${BASH_REMATCH[1]} + local _version=${BASH_REMATCH[2]} + local _type=${BASH_REMATCH[3]} + local _instance=${BASH_REMATCH[4]} + + local _git_repo="" + byref_translate_from_ccm_name2git_repo "${_name}" _git_repo + + local _git_tag="" + byref_translate_from_ccm_version2git_tag "${_version}" _git_tag + + _toString="${_git_repo}-${_git_tag}:$_type:$_instance" + +} + +function byref_translate_from_ccm_4part_query2ccm_4part() { + if [[ -z ${1} ]]; then + echo "Parameter 1 - as ref - cannot be empty" && return 1 + else + local _ccm4part_query=${1} + fi + if [[ -z ${2} ]]; then + echo "Parameter 2 - as ref - cannot be empty" && return 1 + else + local -n _ccm4part=${2} + fi + + [[ "${_ccm4part_query:-}" =~ ${regex_ccm4part} ]] || { + echo "4part does not comply" + return 1 + } + local name=${BASH_REMATCH[1]} + local version=${BASH_REMATCH[2]} + local type=${BASH_REMATCH[3]} + local instance=${BASH_REMATCH[4]} + + _ccm4part=$(ccm query "name match '$name' and version match '$version' and type='$type' and instance='${instance}'" -u -f "%objectname") +} + +function find_n_set_baseline_obj_attrs_from_project(){ + local ccm_project_4part=$1 + local verbose="true" + [[ ${2:-} == "verbose_false" ]] && local verbose="false" + + [[ ${ccm_project_4part} =~ ${regex_ccm4part} ]] || exit 1 + proj_name=${BASH_REMATCH[1]} + proj_version=${BASH_REMATCH[2]} + proj_instance=${BASH_REMATCH[4]} + + project_release=$(ccm properties -f "%release" "${ccm_project_4part}") || return $? + if [[ "$project_release" == "" ]]; then + project_release="void" + release_query="" + else + release_query=" and release='${project_release}'" + fi + + # Find the baseline object of the project with the same release as the project itself + ccm_baseline_obj_and_status_release_this=$(ccm query "has_project_in_baseline('${ccm_project_4part}') ${release_query}" -sby create_time -u -f "%objectname@@@%status@@@%release" | head -1 ) + regex_baseline_attr='^(.+)@@@(.+)@@@(.+)$' + if [[ "${ccm_baseline_obj_and_status_release_this:-}" == "" ]]; then + # No baseline found with primary release tag .. See if other baseline objects are connected ( eg. list any Baseline Object and accept the first ) + ccm_baseline_obj_and_status_release_this=$(ccm query "has_project_in_baseline('${ccm_project_4part}')" -sby create_time -u -f "%objectname@@@%status@@@%release" | head -1 ) + if [[ "${ccm_baseline_obj_and_status_release_this:-}" == "" ]]; then + if [[ "${verbose:-}" == "true" ]]; then + echo "NOTE: No related Baseline Object not found at all: ${ccm_project_4part}" >&2 + fi + else + [[ ${ccm_baseline_obj_and_status_release_this} =~ ${regex_baseline_attr} ]] || exit 1 + ccm_baseline_obj=${BASH_REMATCH[1]} + ccm_baseline_status=${BASH_REMATCH[2]} + ccm_baseline_release=${BASH_REMATCH[3]} + if [[ ${verbose:-} == "true" ]]; then + echo "NOTE: release diff found.. ${ccm_project_4part} / ${project_release} <=> ${ccm_baseline_release} / ${ccm_baseline_obj} - accepted" >&2 + fi + fi + else + [[ ${ccm_baseline_obj_and_status_release_this} =~ ${regex_baseline_attr} ]] || exit 1 + ccm_baseline_obj=${BASH_REMATCH[1]} + ccm_baseline_status=${BASH_REMATCH[2]} + ccm_baseline_release=${BASH_REMATCH[3]} + fi +} + diff --git a/src/main/resources/toGit/migration/sources/ccm/_ccm-start-stop-functions.sh b/src/main/resources/toGit/migration/sources/ccm/_ccm-start-stop-functions.sh new file mode 100644 index 0000000..8efb2fb --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/_ccm-start-stop-functions.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +[[ ${debug:-} == true ]] && set -x + +[[ ${ccm_server_db_path:-} == "" ]] && { + export ccm_server_db_path=//data/ccmdb + echo "Setting the default data path to: ccm_server_db_path=$ccm_server_db_path" + echo " - you can set in your bashrc or similar to overwrite" +} + +[[ ${ccm_server:-} == "" ]] && { + echo "ccm_server is not set.." + echo " - you can set in your bashrc or export it explicitly" + return 1 +} + +function check_ccm_bin_n_set_CCM_HOME() { + echo "Checking if the 'ccm' bin is found: " + export CCM_HOME=$(dirname $(dirname $(which ccm 2> /dev/null ))) || { + echo "ERROR: cannot file 'ccm' in PATH" + return 1 + } + +} + +function check_param() { + if [[ $1 == "" ]]; then + echo "Please set database as parameter 1" + return 2 + fi +} + +function check_user_env_var() { + if [[ ${CCM_USER:-} != "" ]]; then + export ccm_user_option="-n" + echo "Setting ccm user to ${CCM_USER}" + else + echo "Using default user as CM/Synergy user" + fi +} + + +function check_db_started_already() { + echo "Checking if session already started" + if [[ ${ccm_db:-} == "" && ${CCM_ADDR:-} == "" ]]; then + echo "ccm_db and CCM_ADDR is empty - assume ok" + return 0 + fi + if ccm status | grep "No sessions found"; then + return 0 + fi + + if ccm status | grep -A 1 current; then + echo "ERROR: Already have a session running ... " + return 3 + fi +} + +function ccm-start() { + { + check_param $1 || return $? + check_ccm_bin_n_set_CCM_HOME || return $? + check_db_started_already || return $? + check_user_env_var + CCM_ADDR=$(ccm start ${ccm_user_option:-} ${CCM_USER:-} -m -d ${ccm_server_db_path}/${1} -s ${ccm_server} -q) + export CCM_ADDR + echo "CCM_ADDR=${CCM_ADDR}" + + ccm_db=$1 + export ccm_db + + ccm_delim=$(ccm delimiter) + export ccm_delim + echo "ccm_delim=${ccm_delim}" + + ccm_dcm_dbid=$(ccm dcm -show -dbid) + export ccm_dcm_dbid + echo "ccm_dcm_dbid=${ccm_dcm_dbid}" + + } || { unset ccm_db ; return $?; } +} + +function check_db_started_for_stop() { + if [[ "${CCM_ADDR:-}" == "" ]]; then + echo "CCM_ADDR is empty - exit with noop" + unset ccm_db + return 0 + fi + if ccm status | grep "No sessions found"; then + return 0 + fi + if ccm status | grep -A 1 current; then + return 1 + fi +} + +function ccm-stop() { + { + ccm stop || true + echo "unset ccm variables: CCM_HOME CCM_ADDR ccm_db ccm_delim ccm_dcm_dbid" + unset CCM_HOME + unset CCM_ADDR + unset ccm_db + unset ccm_delim + unset ccm_dcm_dbid + } || return $? + +} + +if [[ $- == *i* ]]; then + echo "Synergy is now setup as functions in your bash shell.." + echo "Use:" + echo "- ccm-start " + echo "- ccm-stop" +fi + +#ccm-start $* + +set +x diff --git a/src/main/resources/toGit/migration/sources/ccm/_git-functions.sh b/src/main/resources/toGit/migration/sources/ccm/_git-functions.sh new file mode 100644 index 0000000..cd51a25 --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/_git-functions.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash + +function git_rename_annotated_tag_remote_n_locally(){ + local git_old_tag=$1 + local git_new_tag=$2 + + # reset the committer to get the correct set for the commiting the tag. There is no author of the tag + export GIT_AUTHOR_DATE=$(git tag -l --format="%(taggerdate:iso8601)" ${git_old_tag} | awk -F" " '{print $1 " " $2}') && [[ -z ${GIT_AUTHOR_DATE} ]] && return 1 + export GIT_COMMITTER_DATE=${GIT_AUTHOR_DATE} + export GIT_COMMITTER_NAME=$(git tag -l --format="%(taggername)" ${git_old_tag} ) && [[ -z ${GIT_COMMITTER_NAME} ]] && return 1 + export GIT_COMMITTER_EMAIL=$(git tag -l --format="%(taggeremail)" ${git_old_tag} ) && [[ -z ${GIT_COMMITTER_EMAIL} ]] && return 1 + + echo "Get tag content of: ${git_old_tag}" + git tag -l --format '%(contents)' ${git_old_tag} > ./tag_meta_data.txt + echo "git commit content of ${git_old_tag}" + echo "git tag ${git_new_tag} based on ${git_old_tag}" + git tag -a -F ./tag_meta_data.txt ${git_new_tag} ${git_old_tag}^{} || return 1 + rm -f ./tag_meta_data.txt + git tag --delete ${git_old_tag} || return 1 +} + +function git_delete_tag_on_remote_repos(){ + local git_remote=$1 + local git_tag=$2 + git push ${git_remote} --delete ${git_tag} +} + +function git_create_tag_on_remote_repos(){ + local git_remote=$1 + local git_tag=$2 + git push ${git_remote} ${git_tag} +} + +function git_resolve_tags_wstatus() { + local repo_submodule=$1 + local repo_submodule_rev=$2 + export repo_submodule_rev_wcomponent_wstatus=$(git tag | grep ${repo_submodule}/.*/${repo_submodule_rev}_rel$ || grep_exit=$? ) + # Look then for the "pub" tag + if [[ ${repo_submodule_rev_wcomponent_wstatus} == "" ]]; then + export repo_submodule_rev_wcomponent_wstatus=$(git tag | grep ${repo_submodule}/.*/${repo_submodule_rev}_pub$ || grep_exit=$? ) + else + return 0 + fi + # Accept what is there of remaining + if [[ ${repo_submodule_rev_wcomponent_wstatus} == "" ]]; then + export repo_submodule_rev_wcomponent_wstatus=$(git tag | grep ${repo_submodule}/.*/${repo_submodule_rev}_[dprtis][eueenq][lblsta]$ || grep_exit=$? ) + else + return 0 + fi +} + +function git_initialize_lfs_n_settings() { + echo "Installing Git LFS for the repo" + git lfs install + if [[ ! $( git config --get lfs.locksverify ) ]] ; then + git config --add --local 'lfs.locksverify' false + fi + if [[ ! $( git config --get lfs.contenttype ) ]] ; then + git config --add --local lfs.contenttype 0 + fi +} + +function git_set_execute_bit_in_index_of_extensions() { + # PRE: you are in the repo + exit_code=0 + echo "Based on file extension - set execute bit in the repo on following files:" + git ls-files | grep -ie '.*\.exe$' -ie '.*\.sh$' -ie '.*\.pl$' || exit_code=$? + if [[ ${exit_code} -eq 0 ]] ; then + git ls-files | grep -ie '.*\.exe$' -ie '.*\.sh$' -ie '.*\.pl$' | xargs --no-run-if-empty -d '\n' git update-index --add --chmod=+x + git ls-files | grep -ie '.*\.exe$' -ie '.*\.sh$' -ie '.*\.pl$' | xargs --no-run-if-empty -d '\n' chmod +x + elif [[ ${exit_code} -eq 1 ]] ; then + echo "No files found" + else + echo "ERROR occured" + exit ${exit_code} + fi + echo "Done" +} + +function _git_set_execute_bit(){ + echo Making ${git_ls_file} executable + git update-index --add --chmod=+x ${git_ls_file} + chmod +x ${git_ls_file} +} + +function git_set_execute_bit_in_index_of_unix_tool_file_executable() { + # PRE: you are in the repo + set +x + echo "Based on file reporting 'executable' or 'interpreter' - set execute bit in the repo on following files:" + IFS=$'\r\n' + for git_ls_file in `git ls-files`; do + if file ${git_ls_file} |grep executable 2>&1 > /dev/null; then + _git_set_execute_bit + elif file ${git_ls_file} |grep interpreter 2>&1 > /dev/null; then + _git_set_execute_bit + fi + done + [[ ${debug:-} == "true" ]] && set -x + echo "Done" +} + +function git_find_n_fill_empty_dirs_gitignore() { + echo "Fill empty directories with .gitignore" + file_empty_dirs_tmp="empty_dirs.tmp" + local exit_code=0 + /usr/bin/find . -mindepth 1 -type d -empty | grep -v '\./\.git/' > ${file_empty_dirs_tmp} || exit_code=$? + if [[ ${exit_code} -eq 0 ]] ; then + # grep did give an output - hence there are empty directories + IFS=$'\r\n' + while read empty_dir; do + echo ${empty_dir} + cp ${script_dir}/emptydir.gitignore ${empty_dir}/.gitignore + done < ${file_empty_dirs_tmp} || exit 1 + unset IFS + elif [[ ${exit_code} -eq 1 ]] ; then + # grep did not give an output - hence there are no empty directories + echo "INFO: no empty directories where found" + else + echo "ERROR occured" + exit ${exit_code} + fi + rm -f ${file_empty_dirs_tmp} + echo "Done" +} + +function git_copy_tag(){ + local source_tag="$1" + local target_tag="$2" + local GIT_COMMITTER_NAME="$(git tag -l --format=\"%\(taggername\)\" \"$source_tag\" )" + local GIT_COMMITTER_EMAIL="$(git tag -l --format=\"%\(taggeremail\)\" \"$source_tag\" )" + local GIT_COMMITTER_DATE="$(git tag -l --format=\"%\(taggerdate:iso8601\)\" \"$source_tag\" )" + git tag \ + "$target_tag" \ + "$(git tag -l --format=\"%\(*objectname\)\" \"$source_tag\" )" \ + -a \ + -m "$(git tag -l --format=\"%\(contents\)\" \$source_tag\" )" +} \ No newline at end of file diff --git a/src/main/resources/toGit/migration/sources/ccm/ccm-baseline-history-get-root.sh b/src/main/resources/toGit/migration/sources/ccm/ccm-baseline-history-get-root.sh new file mode 100644 index 0000000..b7d3cbe --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/ccm-baseline-history-get-root.sh @@ -0,0 +1,48 @@ +#!/bin/bash +[[ ${debug:-} == true ]] && set -x +set -u +set -e + +source $(dirname $0)/_ccm-functions.sh || source ./_ccm-functions.sh + +if [[ ${1} == "" ]]; then + printf "project parameter 1 is empty - skip" 1>&2 + exit 0 +fi +BASELINE_PROJECT_tmp="$1" +BASELINE_PROJECT="$1" + + +printf "Processing ${BASELINE_PROJECT}\n" 1>&2 + +until [[ "${BASELINE_PROJECT:-}" == "" ]] ; do + this_project4part="${BASELINE_PROJECT}" + + this_project_name=$(echo ${BASELINE_PROJECT} | awk -F"${ccm_delim}" '{print $1}') + + query="is_baseline_project_of('${BASELINE_PROJECT}')" + BASELINE_PROJECT=$(ccm query "is_baseline_project_of('${BASELINE_PROJECT}')" -u -f "%objectname") || BASELINE_PROJECT="" + baseline_name=`printf "${BASELINE_PROJECT:-}" | awk -F"${ccm_delim}" '{print $1}'` + if [[ "${baseline_name:-}" != "" && "${baseline_name}" != "${this_project_name}" ]]; then + printf "Stop traversing - name changed: '${this_project_name}' -> '$baseline_name'\n\n" 1>&2 + printf "Get sucessors with '${this_project_name}' of baseline_project '${BASELINE_PROJECT}' (different name) as well\n\n" 1>&2 + echo "${this_project4part}" + ccm query "has_baseline_project('${BASELINE_PROJECT}') and name='${this_project_name}' " -u -f "%objectname" + exit + fi + if [[ "${BASELINE_PROJECT:-}" != "" ]] ; then + printf "${BASELINE_PROJECT} -> " 1>&2 + else + printf " - baseline empty\n\n" 1>&2 + break + fi +done +project_status=$(ccm attr -show status "${this_project4part}") +case "$project_status" in + integrate|released|sqa|test) + printf "${this_project4part}" + ;; + *) + printf "Project $this_project4part is in state: $project_status - Skip - do not list" 1>&2 + ;; +esac diff --git a/src/main/resources/toGit/migration/sources/ccm/ccm-baseline-history.sh b/src/main/resources/toGit/migration/sources/ccm/ccm-baseline-history.sh new file mode 100644 index 0000000..0e1d1a4 --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/ccm-baseline-history.sh @@ -0,0 +1,142 @@ +#!/bin/bash + +set -euo pipefail + +[[ ${debug:-} == "true" ]] && set -x + + +use_wildcard="" # * + +# Load functions +source $(dirname $0)/_ccm-functions.sh || source ./_ccm-functions.sh + +find_project_baseline_to_convert(){ + local CURRENT_PROJECT=$1 + local inherited_string_local=$2 + + if [[ "${use_wildcard}" == "*" ]]; then + proj_name=$(printf "${CURRENT_PROJECT}" | awk -F"${ccm_delim}|:" '{print $1}') + proj_version=$(printf "${CURRENT_PROJECT}" | awk -F"${ccm_delim}|:" '{print $2}') + proj_instance=$(printf "${CURRENT_PROJECT}" | awk -F"${ccm_delim}|:" '{print $4}') + query="has_baseline_project(name match '${proj_name}*' and version='${proj_version}' and type='project' and instance='${proj_instance}') and ( status='integrate' or status='test' or status='sqa' or status='released' )" + else + [[ ${CURRENT_PROJECT:-} =~ ${regex_ccm4part} ]] || { + echo "4part does not comply" + return 1 + } + local proj_name=${BASH_REMATCH[1]} + local version=${BASH_REMATCH[2]} + local type=${BASH_REMATCH[3]} + local instance=${BASH_REMATCH[4]} + + query="has_baseline_project('${CURRENT_PROJECT}') and project='$proj_name' and ( status='integrate' or status='test' or status='sqa' or status='released' )" + fi + + # All status versions + [[ ${debug:-} == "true" ]] && { + printf "_________________________\n%s\n" "$(ccm query "${query}" -u -f "%objectname")" >&2 + } + IFS=$'\r\n' + for SUCCESSOR_PROJECT in $(ccm query "${query}" -u -f "%objectname") ; do + local inherited_string="${inherited_string_local} -> ${CURRENT_PROJECT}" + [[ ${debug:-} == "true" ]] && printf "${inherited_string}\n" >&2 + if [[ $(ccm properties -f %ccm2git_migrate "${SUCCESSOR_PROJECT}" ) == "FALSE" ]]; then + echo "SKIP: ${SUCCESSOR_PROJECT} ccm2git_migrate=FALSE - continue" >&2 + continue # Next if already for some odd reason exists - seen in firebird~BES-SW-0906-1.8:project:2 + fi + if [[ `grep "$SUCCESSOR_PROJECT@@@$CURRENT_PROJECT" ${projects_file}` ]]; then + echo "ALREADY include in project file - continue" >&2 + continue # Next if already for some odd reason exists - seen in firebird~BES-SW-0906-1.8:project:2 + fi + exit_code="0" + find_n_set_baseline_obj_attrs_from_project "${SUCCESSOR_PROJECT}" "verbose_true" || exit_code=$? + if [[ "${exit_code}" != "0" ]] ; then + echo "ERROR: Project not found: ${SUCCESSOR_PROJECT}" + exit ${exit_code} + fi + if [[ ${ccm_baseline_status:-} == "test_baseline-DONT_CARE" ]] ; then + # Figure out if the project is in use as as baseline in and other project + exit_code=0 + project_baseline_childs=$(ccm query "has_baseline_project('${SUCCESSOR_PROJECT}') and ( status='integrate' or status='test' or status='sqa' or status='released' )" -u -f "%objectname" | head -1 ) || exit_code=$? + if [[ $exit_code -eq 6 ]]; then + project_baseline_childs="" + elif [[ $exit_code -ne 0 ]]; then + echo "ERROR: something when wrong: exit code: $exit_code" + exit $exit_code + fi + if [[ "${project_baseline_childs:-}" != "" ]]; then + echo "ACCEPT: Related Baseline Object is in test status: ${SUCCESSOR_PROJECT}: ${ccm_baseline_obj_and_status_release_this} - but at least in use as baseline of project: ${project_baseline_childs}" >&2 + else + if [[ $(ccm finduse -all_projects "$(echo ${SUCCESSOR_PROJECT} )" | grep "Object is not used in scope." ) ]]; then + # not in use as sub project" + echo "SKIP: Related Baseline Object is in test status and is NOT in use as subproject: ${SUCCESSOR_PROJECT}: ${ccm_baseline_obj_and_status_release_this} - and is leaf in project baseline history" >&2 + continue + else + # in use + echo "ACCEPT: Related Baseline Object is in test status and is in use as subproject: ${SUCCESSOR_PROJECT}: ${ccm_baseline_obj_and_status_release_this} - even is leaf in project baseline history" >&2 + fi + fi + fi + regex_revision_contains_History='^.*\.History.*$' + if [[ ${proj_version:-} =~ ${regex_revision_contains_History} ]] ; then + exit_code=0 + project_baseline_childs=$(ccm query "has_baseline_project('$(echo ${SUCCESSOR_PROJECT} )') and ( status='integrate' or status='test' or status='sqa' or status='released' )" -u -f "%objectname" | head -1 ) || exit_code=$? + if [[ $exit_code -eq 6 ]]; then + project_baseline_childs="" + elif [[ $exit_code -ne 0 ]]; then + echo "ERROR: something when wrong: exit code: $exit_code" + exit $exit_code + fi + if [[ "${project_baseline_childs:-}" != "" ]]; then + echo "ACCEPT: Project revision contains 'History': ${SUCCESSOR_PROJECT}: ${ccm_baseline_obj_and_status_release_this} - but is in use as baseline of project: ${project_baseline_childs}" >&2 + else + if [[ $(ccm finduse -all_projects "$(echo ${SUCCESSOR_PROJECT} )" | grep "Object is not used in scope." ) ]]; then + # not in use as sub project" + echo "SKIP: Project revision contains 'History and is NOT in use as subproject: ${SUCCESSOR_PROJECT}: ${ccm_baseline_obj_and_status_release_this} - even is leaf in project baseline history" >&2 + continue + else + # in use + echo "ACCEPT: Project revision contains '.History', but is in use as a subproject: ${SUCCESSOR_PROJECT}: ${ccm_baseline_obj_and_status_release_this} - even is leaf in project baseline history" >&2 + fi + fi + fi + git_CURRENT_PROJECT="" + byref_translate_from_ccm_4part2git_repo_4part "${CURRENT_PROJECT}" git_CURRENT_PROJECT + + git_SUCCESSOR_PROJECT="" + byref_translate_from_ccm_4part2git_repo_4part "${SUCCESSOR_PROJECT}" git_SUCCESSOR_PROJECT + printf "${git_SUCCESSOR_PROJECT}@@@${git_CURRENT_PROJECT}@@@${SUCCESSOR_PROJECT}@@@${CURRENT_PROJECT}\n" >> ${projects_file} + find_project_baseline_to_convert "${SUCCESSOR_PROJECT}" "${inherited_string}" + done +} + +## MAIN ## +git_BASELINE_PROJECT="$1" + +export projects_file="./projects.txt" +#rm -f ${projects_file} +if [ "${use_cached_project_list:-}X" == "trueX" ]; then + if [ -e ${projects_file} ] ; then + cat ${projects_file} + exit 0 + fi +fi + +ccm_BASELINE_PROJECT="" +byref_translate_from_git_repo_4part2ccm_4part "${git_BASELINE_PROJECT}" ccm_BASELINE_PROJECT + + +checked_version=$(ccm properties -f %version "$ccm_BASELINE_PROJECT" ) || { + exit_code=$? + echo "Project: $ccm_BASELINE_PROJECT does not exit - exit 1" + exit $exit_code +} + +init_project_name=$(printf "${ccm_BASELINE_PROJECT}" | awk -F"${ccm_delim}" '{print $1}') +instance=$(printf "${ccm_BASELINE_PROJECT}" | awk -F"${ccm_delim}|:" '{print $4}' ) + +inherited_string="${ccm_BASELINE_PROJECT}" +echo "$git_BASELINE_PROJECT@@@${init_project_name}${ccm_delim}init:project:${instance}@@@$ccm_BASELINE_PROJECT@@@${init_project_name}${ccm_delim}init:project:${instance}" > ${projects_file} + +find_project_baseline_to_convert "${ccm_BASELINE_PROJECT}" "${inherited_string}" +cat ${projects_file} diff --git a/src/main/resources/toGit/migration/sources/ccm/ccm-convert-flat2submodules-existingRepoHistory.sh b/src/main/resources/toGit/migration/sources/ccm/ccm-convert-flat2submodules-existingRepoHistory.sh new file mode 100644 index 0000000..77a0c7f --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/ccm-convert-flat2submodules-existingRepoHistory.sh @@ -0,0 +1,725 @@ +#!/usr/bin/env bash +[[ ${debug:-} == "true" ]] && set -x +[[ ${run_local:-} == "true" ]] && push_remote="false" +set -u +set -e +set -o pipefail + +# Load functions +source $(dirname $0)/_ccm-functions.sh || source ./_ccm-functions.sh +source $(dirname $0)/_git-functions.sh || source ./_git-functions.sh + +execution_root_directory=$(pwd) + +# parameter $1 is the project list file generated by the baseline_history.sh script +export repo_name=${1} + +export repo_init_tag=${2} +export repo_submodules=${3} +[[ ${submodule_update_mode:-} == "" ]] && export submodule_update_mode="directory" # or update-index which is old style +[[ ${push_tags_in_submodules:-} == "" ]] && export push_tags_in_submodules="false" +[[ "${execute_mode:-}" == "" ]] && export execute_mode="normal" + +export gitrepo_project_original=${4} +export project_instance=${5} +export gitignore_path_n_files=${6} # :@:.. +export gitattributes_path_n_files=${7:-} # :@:.. + +export ccm_name="" +byref_translate_from_git_repo2ccm_name "${repo_name}" "$project_instance" ccm_name + +declare -A repo_submodules_map +if [[ $(echo "${repo_submodules}" | grep "," ) ]]; then + IFS="," +else + IFS=" " +fi +for repo_submodule_from_param in $(echo "${repo_submodules}"); do + repo_submodule_raw_name=$(echo ${repo_submodule_from_param} | awk -F ":" '{print $1}') + repo_submodules_map["${repo_submodule_raw_name}"]="${repo_submodule_raw_name}" +done +unset repo_submodule_raw_name +unset repo_submodule_from_map +unset IFS + +#export project_revisions=`cat ${1}` + +if [[ "${BITBUCKET_PROD_USERNAME:-}" != "" && "${BITBUCKET_PROD_PASSWORD:-}" != "" ]]; then + http_remote_credentials="${BITBUCKET_PROD_USERNAME}:${BITBUCKET_PROD_PASSWORD}@" +else + http_remote_credentials="" +fi + +export git_ssh_remote=ssh://git@${git_server_path}/${repo_name}.git +export git_ssh_remote_orig=ssh://git@${git_server_path}/${repo_name}_orig.git +export git_https_remote=$(echo ${git_ssh_remote} | sed -e "s/ssh:\/\/git@/https:\/\/${http_remote_credentials}/" -e 's/7999/7990\/scm/' | sed -e 's/ssh-//' | sed -e 's/:7990//') +export git_https_remote_orig=$(echo ${git_ssh_remote_orig} | sed -e "s/ssh:\/\/git@/https:\/\/${http_remote_credentials}/" -e 's/7999/7990\/scm/' | sed -e 's/ssh-//' | sed -e 's/:7990//' ) +export git_remote_to_use=${git_https_remote} +export git_remote_to_use_orig=${git_https_remote_orig} +echo "INFO: Use remote : ${git_remote_to_use} and ${git_remote_to_use_orig}" + +function resolve_git_common_target_repo { + local dir_name_from_git_common_target_repo=$(dirname ${git_common_target_repo} ) + if [[ $dir_name_from_git_common_target_repo == "." ]]; then + echo "The git_common_target_repo does not include relative prefix - use it directly - i.e. same project - replace repo " + local server_base_url=$(dirname "$git_remote_to_use") + export https_remote_common=${server_base_url}/${git_common_target_repo} + else + echo "The git_common_target_repo contains relative prefix - use it directly with / - i.e. same project" + local server_base_url=$(dirname $(dirname "$git_remote_to_use")) + export https_remote_common="${server_base_url}/${git_common_target_repo}" + fi +} + +if [[ "${git_common_target_repo:-}" != "" ]]; then + + resolve_git_common_target_repo + + echo "INFO: Configured to common repo: ${git_common_target_repo} : ${https_remote_common} " + echo " - consider to enable 'push_to_remote_during_conversion=true'" + echo " to avoid false failed fetch of tags in case of history rewrite" +fi + +function convert_revision(){ + local root_dir=$(pwd) + local repo_convert_rev_tag=$1 + + local tag_to_convert=$(git tag | grep "${repo_name}/.*/${repo_convert_rev_tag}$" || grep_ext_value=$?) + + set +x + if [[ "${tag_to_convert}" == "" ]] ; then + echo "============================================================================" + echo " BEGIN: ${repo_convert_rev_tag}" + echo "============================================================================" + [[ ${debug:-} == "true" ]] && set -x + else + echo "=====================================================================================================" + echo " Already done - skip: ${repo_convert_rev_tag} -> ${tag_to_convert}" + echo "=====================================================================================================" + [[ ${debug:-} == "true" ]] && set -x + return + fi + local ccm_repo_convert_rev_tag=${repo_convert_rev_tag:: -4} + + local exit_code="0" + git_4part="${repo_name}${ccm_delim}${ccm_repo_convert_rev_tag}:project:${project_instance}" + + ccm_4part="" + byref_translate_from_git_repo_4part2ccm_4part "${git_4part}" ccm_4part + + find_n_set_baseline_obj_attrs_from_project "${ccm_4part}" "verbose_false" || exit_code=$? + if [[ "${exit_code}" != "0" ]] ; then + echo "ERROR: Project not found: ${ccm_4part}" + exit ${exit_code} + fi + local ccm_release=$(echo ${project_release} | cut -d "/" -f 2) # inherited from function find_n_set_baseline_obj_attrs_from_project + + [[ "${ccm_release:-}" == "x" ]] && ( echo "Release is empty!!" && exit 1) + [[ "${ccm_release:-}" == "" ]] && ( echo "Release is !!" && exit 1) # if the release is it is rewritten to void as BitBucket does not allow <> chars in tags + + #NOTE: The next line is suppressing the support for having a baseline project with a different name than is being converted: ( and name='${repo_name}' ) + local ccm_baseline_from_tag_info="$(ccm query "is_baseline_project_of('${ccm_4part}') and name='${ccm_name}'" \ + -u -f "%version" )" || return 1 + + + if [[ "${ccm_baseline_from_tag_info}" != "" ]] ; then + # prefer released if found + baseline_from_tag_info="" + byref_translate_from_ccm_version2git_tag "$ccm_baseline_from_tag_info" baseline_from_tag_info + + local repo_baseline_rev_tag_wcomponent_wstatus=$(git tag | grep ^${repo_name}/.*/${baseline_from_tag_info}_rel$ || grep_ext_value=$? ) + if [[ "${repo_baseline_rev_tag_wcomponent_wstatus}" == "" ]]; then + local repo_baseline_rev_tag_wcomponent_wstatus=$(git tag | grep ^${repo_name}/.*/${baseline_from_tag_info}_[dprtis][eueenq][lblsta]$ || grep_ext_value=$? ) + tag_amounts=$(echo ${repo_baseline_rev_tag_wcomponent_wstatus} | wc -l ) + [[ ${tag_amounts} -gt 1 ]] && { printf "ERROR: More than one tag (${tag_amounts}) found in repo_baseline_rev_tag_wcomponent_wstatus\n${repo_baseline_rev_tag_wcomponent_wstatus}\n" && return 1 ;} + fi + if [[ "${repo_baseline_rev_tag_wcomponent_wstatus}" == "" ]] ; then + #find the original tag and convert it first + local repo_baseline_orig_tag_wstatus=$(git tag | grep "^${baseline_from_tag_info}_[dprtis][eueenq][lblsta]$" || grep_ext_value=$?) + if [[ "${repo_baseline_orig_tag_wstatus}" != "" ]]; then + echo "INFO: RECURSIVE action - Wait with ${ccm_repo_convert_rev_tag} / baseline_from_tag_info_wstatus is empty hence the order of tags of the single commit is listed in the wrong order of tags on same commit" + convert_revision ${repo_baseline_orig_tag_wstatus} + # reset it to make sure the recursive have not reset it + local repo_baseline_rev_tag_wcomponent_wstatus=$(git tag | grep ^${repo_name}/.*/${baseline_from_tag_info}_[dprtis][eueenq][lblsta]$ || grep_ext_value=$? ) + set +x + echo "============================================================================" + echo " CONTINUE: ${repo_convert_rev_tag}" + echo "============================================================================" + [[ ${debug:-} == "true" ]] && set -x + else + echo "ERROR: Dont know why we ended here - something is not right!!" + echo "The baseline tag that is needed for the conversion of tag: ${repo_convert_rev_tag} cannot even find the tag unconverted: ${baseline_from_tag_info}" + return 1 + fi + fi + else + local repo_baseline_rev_tag_wcomponent_wstatus="${repo_name}/${repo_init_tag}/${repo_init_tag}" + fi + if [[ -f ${execution_root_directory}/${ccm_db}_gitfiles/baseline_history_rewrite.txt ]]; then + # lookup in the baseline history rewrite file + repo_baseline_rev_tag_wcomponent_wstatus_lookup=$(grep -E "^${repo_baseline_rev_tag_wcomponent_wstatus}@.+/.+/.+$" ${execution_root_directory}/${ccm_db}_gitfiles/baseline_history_rewrite.txt | cut -d @ -f 2- ) || { echo "INFO: No rewriting baseline found - skip" ; } + if [[ ${repo_baseline_rev_tag_wcomponent_wstatus_lookup:-} != "" ]]; then + local tag_found=false + local tag_fetch_continue=true + local total_sleep_sec=0 + local sleep_sec=300 + local sleep_total_wait=86400 # 24 hours + while $tag_fetch_continue ; do + # TODO: consider to check if the baseline has changed state like pub -> rel + if git fetch ${https_remote_common} -f --no-tags refs/tags/${repo_baseline_rev_tag_wcomponent_wstatus_lookup}:refs/tags/${repo_baseline_rev_tag_wcomponent_wstatus_lookup} ; then + echo "All good - we got the tag" + [[ -d .git/lfs ]] && { + git lfs fetch ${https_remote_common} --all + git lfs fetch ${git_remote_to_use} --all + } + tag_found=true + tag_fetch_continue=false + else + if [[ ${total_sleep_sec} -gt ${sleep_total_wait} ]]; then + echo "ERROR: We waited for 24 hours - stop - exit" + exit 1 + fi + echo "Sleep: ${sleep_sec} sec of total allowed: ${sleep_total_wait} sec ( 24 hours )" + sleep $sleep_sec + total_sleep_sec=$(( total_sleep_sec + sleep_sec)) + fi + done + repo_baseline_rev_tag_wcomponent_wstatus=${repo_baseline_rev_tag_wcomponent_wstatus_lookup} + fi + fi + repo_baseline_rev_tag_wcomponent_wstatus_gitnormalized="" + byref_translate_from_ccm_version2git_tag "${repo_baseline_rev_tag_wcomponent_wstatus}" repo_baseline_rev_tag_wcomponent_wstatus_gitnormalized + echo "repo_baseline_rev_tag_wcomponent_wstatus_gitnormalized=${repo_baseline_rev_tag_wcomponent_wstatus_gitnormalized}" + + local repo_convert_rev_tag_wcomponent_wstatus="${repo_name}/${ccm_release}/${repo_convert_rev_tag}" + repo_convert_rev_tag_wcomponent_wstatus_gitnormalized="" + byref_translate_from_ccm_version2git_tag "${repo_convert_rev_tag_wcomponent_wstatus}" repo_convert_rev_tag_wcomponent_wstatus_gitnormalized + echo "repo_convert_rev_tag_wcomponent_wstatus_gitnormalized=${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" + + # Get the right content + if [ $(git describe ${repo_convert_rev_tag}) ] ; then + # we do have the correct 'content' tag checkout it out + pwd + git clean -xffd || git clean -xffd || git clean -xffd # It can happen that the first clean fails, but more tries can fix it + for path_failed_to_remove in $(git reset -q --hard ${repo_convert_rev_tag} 2>&1 | awk -F "'" '{print $2}'); do + echo "Reset/remove submodule and it's path: ${path_failed_to_remove}" + git rm -rf --cached ${path_failed_to_remove} > /dev/null 2>&1 || echo "never mind" + rm -rf ${path_failed_to_remove} + done + else + # we do not have the 'content' tag available - investigate its history if it exists ( e.g. missing in repo ) + ./ccm-baseline-history-get-root.sh "${ccm_name}${ccm_delim}${ccm_repo_convert_rev_tag}:project:${project_instance}" + exit 1 + fi + + [[ ${repo_baseline_rev_tag_wcomponent_wstatus_gitnormalized} == "" ]] && { + echo "ERROR: repo_baseline_rev_tag_wcomponent_wstatus_gitnormalized is empty - something is wrong" + exit 1 + } + # Move the workarea pointer to the 'baseline' tag + git reset -q --mixed ${repo_baseline_rev_tag_wcomponent_wstatus_gitnormalized} > /dev/null 2>&1 + if [[ "${repo_baseline_rev_tag_wcomponent_wstatus_lookup:-}" != "" ]]; then + # we dont the tag after we have reset to it - delete it again to avoid push confusion + git tag --delete "${repo_baseline_rev_tag_wcomponent_wstatus_lookup}" + fi + + # get the .gitignore files from init commit + for file in $(git ls-tree --name-only -r ${repo_name}/init/init^{}); do + git checkout ${repo_name}/init/init $file + git add ${file} + done + + exit_code=0 + if [[ ${submodules_from_baseline_obj:-} == true ]] ; then + ccm_submodules4part="$(ccm query "is_project_in_baseline_of(has_project_in_baseline('${ccm_4part}')) and not ( name='${repo_name}' )" -u -f "%objectname" )" || exit_code=$? + if [[ $exit_code -ne 0 ]]; then + if [[ $exit_code -eq 6 ]]; then + # query did not give outout - try to find the previous release tag via git describe and get setup from there + # previous _rel + echo "WARNING: No submodules found.. Restore what was already there from previous previous submodules list and content" + if git restore .gitmodules ; then + regex_submodule_line='^160000[[:blank:]]commit[[:blank:]]([0-9a-f]{40})[[:blank:]]+(.+)$' + IFS=$'\n\r' + for submodule_line in $(git ls-tree -r HEAD | grep -E '^160000[[:blank:]]+commit[[:blank:]]+[0-9a-f]{40}[[:blank:]]+.+$') ; do + echo "Process: $submodule_line" + if [[ ${submodule_line} =~ ${regex_submodule_line} ]] ; then + local submodule_sha1=${BASH_REMATCH[1]} + local submodule_path=${BASH_REMATCH[2]} + git restore ${submodule_path} + #git update-index --add --replace --cacheinfo "160000,${submodule_sha1},${submodule_path}" + unset submodule_sha1 + unset submodule_path + else + echo skip: $submodule_line +# cat .gitmodules +# exit 1 + fi + done + else + echo "INFO: no .gitmodules found" + fi + else + echo "ERROR: Something went wrong" + exit 1 + fi + fi + else + rm -f .gitmodules + if [[ ! ${repo_submodules} == "" ]]; then + touch .gitmodules + fi + ccm_submodules4part="$(ccm query "is_member_of('${ccm_4part}') and name!='${ccm_name}' and type='project'" -u -f "%objectname" )" || exit_code=$? + fi + if [[ $exit_code -ne 0 ]]; then + if [[ $exit_code -eq 6 ]]; then + # query did not give outout + repo_submodules4part="" + else + echo "ERROR: Something went wrong" + exit 1 + fi + fi + IFS=$'\n\r' + for ccm_submodule4part in ${ccm_submodules4part} ; do + set +x + [[ ${ccm_submodule4part} =~ ${regex_ccm4part} ]] || exit 1 + local ccm_submodule_name=${BASH_REMATCH[1]} + local ccm_submodule_rev=${BASH_REMATCH[2]} + local ccm_submodule_inst=${BASH_REMATCH[4]} + + local repo_submodule_name="" + byref_translate_from_ccm_name2git_repo "$ccm_submodule_name" repo_submodule_name + local repo_submodule_rev="" + byref_translate_from_ccm_version2git_tag "$ccm_submodule_rev" repo_submodule_rev + + # Lookup the subproject if present + repo_submodule=$(echo ${repo_submodules_map[${repo_submodule_name:-}]:-}) + if [[ "${repo_submodule}" == "" ]] ; then + echo "[INFO]: ${repo_submodule_name} - The subproject not found in projects to add as submodules - skip" + [[ ${debug:-} == "true" ]] && set -x + cd ${root_dir} + continue + fi + unset repo_submodule_name + echo "[INFO]: ${ccm_submodule4part} - use it" + [[ ${debug:-} == "true" ]] && set -x + + git_remote_submodule_to_use=$(echo ${git_remote_to_use} | sed -e "s/\/${repo_name}.git/\/${repo_submodule}.git/") + if [[ ${submodules_from_baseline_obj:-} == true ]] ; then + shared_config_file=$(git ls-tree -r --name-only ${repo_convert_rev_tag} | grep '^.*/shared_config.txt$') || { + if [[ $? == 1 ]] ; then + echo "[INFO]: shared_config.txt file not found - ok" + else + echo "[ERROR]: Something went wrong in finding the shared_config.txt file" + exit 1 + fi + } + if [[ "${shared_config_file:-}" != "" ]]; then + echo "[INFO]: shared_config.txt found in the git tag ${repo_convert_rev_tag}" + if ! git cat-file -p ${repo_convert_rev_tag}:$(git ls-tree -r --name-only ${repo_convert_rev_tag} | grep '^.*/shared_config.txt$' ) | grep -E "^${ccm_submodule_name}${ccm_delim}.+$|^\.\.\\\\(.+\\\\)*${ccm_submodule_name}${ccm_delim}.+$" ; then + echo "WARNING: The submodule is not to be found in shared_config_file - fall-back to default" + _path_from_shared_config_file="." + else + # The submodule reference found in shared_config file - convert to unix slashes + _path_from_shared_config_file=$(dirname $(git cat-file -p ${repo_convert_rev_tag}:$(git ls-tree -r --name-only ${repo_convert_rev_tag} | grep '^.*/shared_config.txt$') \ + | grep -E "^${ccm_submodule_name}${ccm_delim}.+$|^\.\.\\\\(.+\\\\)*${ccm_submodule_name}${ccm_delim}.+$" | sed -e 's/\\/\//g')) + fi + if [[ ${_path_from_shared_config_file} == "${ccm_submodule_name}" || ${_path_from_shared_config_file} == "." ]]; then + echo "The path was not explicitly specified in the shared_config file or module not found - add it to the dir of the shared_config.txt" + git_submodule_path="$(dirname "${shared_config_file}" | sed -e 's/\\/\//g' )/${repo_submodule}" + else + # We found a qualified path for the submodule in the shared_config file + echo "Use the found path from ${shared_config_file}: ${_path_from_shared_config_file}" + git_submodule_path="$( realpath -m --relative-to=./ $(dirname ${shared_config_file})/${_path_from_shared_config_file})/${repo_submodule}" + echo "Set path of submodule to $git_submodule_path in root folder" + fi + else + echo "Place the submodule in the root folder as fall-back" + git_submodule_path=${repo_submodule} + fi + else + git_submodule_path_in_project=$(ccm finduse -p "${ccm_submodule4part}" | grep "^[[:space:]]" | grep -v '[[:space:]]Projects:' | grep "${ccm_submodule4part//:project:1/''}" | sed -e 's/\t//g' | sed -e 's/\\/\//g' | grep -e "^.*@${ccm_4part//:project:1/''}"'$' | awk -F "${ccm_delim}" '{print $1}' | sed -e "s/^${repo_name}/./g" | sed -e "s/\/${repo_submodule}//") + git_submodule_path=${git_submodule_path_in_project}/${repo_submodule} + fi + [[ ${git_submodule_path:-} == "" ]] && ( echo "submodule path is empty - exit 1" && exit 1 ) + + case ${submodule_update_mode:-} in + "update-index") + git add -A . # just add here so execute bit can be manipulated in staged + # Get the sha1 from a reference / tag or reference is sha1 as not provided + https_remote_submodule=$(echo ${git_remote_to_use} | sed -e "s/\/me_limited_submodules.git/\/${repo_submodule}.git/") + export repo_submodule_sha1=$(git ls-remote --tag ${https_remote_submodule} | grep -e "${repo_submodule}/.*/${repo_submodule_rev}_rel$" | awk -F" " '{print $1}') + # Look then for the "pub" tag + [[ ${repo_submodule_sha1} == "" ]] && export repo_submodule_sha1=$(git ls-remote --tag ${https_remote_submodule} | grep -e "${repo_submodule}/.*/${repo_submodule_rev}_pub$" | awk -F" " '{print $1}') + # Accept what is there of remaining + [[ ${repo_submodule_sha1} == "" ]] && export repo_submodule_sha1=$(git ls-remote --tag ${https_remote_submodule} | grep -e "${repo_submodule}/.*/${repo_submodule_rev}_[dprtis][eueenq][lblsta]$" | awk -F" " '{print $1}') + + [[ ${repo_submodule_sha1} == "" ]] && exit 1 + + echo "INFO: Setting submodule: ${repo_submodule} to ${repo_submodule_sha1}" + + echo "INFO: Remove the old dir in git index" + if [[ ! $(git rm -rf ${repo_submodule} > /dev/null 2>&1 ) ]]; then + rm -rf ${repo_submodule} + fi + echo "INFO: Add the submodule to .gitmodules" + git config -f ./.gitmodules --add submodule.${repo_submodule}.path ${repo_submodule} + git config -f ./.gitmodules --add submodule.${repo_submodule}.url ../${repo_submodule}.git + git add ./.gitmodules + + echo "INFO: set the sha1: ${repo_submodule_sha1} as the reference of the submodule: ${repo_submodule}" + git update-index --add --replace --cacheinfo "160000,${repo_submodule_sha1},${repo_submodule}" + if [[ ${push_tags_in_submodules} == "true" ]]; then + #TODO: if push of tags in submodules is desired + exit 1 + https_url=$(git config --get remote.origin.url | awk -F "/" '{print $3}' | sed -e 's/git@/https:\/\//g' -e 's/7999/7990/') + curl https://api.bitbucket.org/2.0/repositories/jdoe/myrepo/refs/tags \ + -s -u jdoe -X POST -H "Content-Type: application/json" \ + -d '{ "name" : "new-tag-name", "target" : { "hash" : "a1b2c3d4e5f6" } }' + fi + git status | grep "${repo_submodule}" + cat .gitmodules + ;; + "directory") + if [[ ! $(git rm -rf ${repo_submodule}) ]]; then + rm -rf ${repo_submodule} + # This should really not be necessary # rm -rf .git/modules/${repo_submodule} + fi + if [[ ! $(git submodule update --init --recursive --force ${repo_submodule}) ]] ; then + git rm -rf ${git_submodule_path} --cached || echo "Good already - never mind" + rm -rf ${git_submodule_path} + touch .gitmodules && git add .gitmodules + if ! git submodule add --force --name "${repo_submodule}" "../${repo_submodule}.git" "${git_submodule_path}" ; then + cd ${git_submodule_path} + git remote -v + git status + pwd + git checkout -B master + git reset --hard ${repo_submodule}/${repo_init_tag}/${repo_init_tag} || { + git fetch origin +refs/tags/*:refs/tags/* + git reset --hard ${repo_submodule}/${repo_init_tag}/${repo_init_tag} + } + cd ${root_dir} + git submodule add --force --name "${repo_submodule}" "../${repo_submodule}.git" "${git_submodule_path}" + else + git submodule add --force --name "${repo_submodule}" "../${repo_submodule}.git" "${git_submodule_path}" + fi + fi + git add ./.gitmodules + + cd ${git_submodule_path} + + # Look for the "rel" tag first + git fetch ${git_remote_submodule_to_use} --tags +refs/heads/*:refs/remotes/origin/* + git_resolve_tags_wstatus "${repo_submodule}" "${repo_submodule_rev}" + if [[ "${repo_submodule_rev_wcomponent_wstatus}" == "" ]] ; then + echo "[ERROR]: Could find the revision ${repo_submodule}/.*/${repo_submodule_rev}_???" + exit 1 + fi + + if [[ ${push_tags_in_submodules} == "true" ]]; then + # root project tag handling + if [[ ! $(git describe ${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}) ]] ; then + # it was not found try and fetch to make 100% sure for whatever reason it is not here.. + git fetch ${git_remote_submodule_to_use} --tags +refs/heads/*:refs/remotes/origin/* + fi + if [[ $(git describe ${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}) ]] ; then + # we already have the correct tag, so just set it and move on.. + git reset -q --hard ${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized} + git clean -xffd + unset repo_submodule_rev + unset ccm_submodule_inst + cd ${root_dir} + continue + fi + fi + + if [ $(git describe "${repo_submodule_rev_wcomponent_wstatus}") ] ; then + # we do have the correct 'content' tag - reset hard to it and make sure we are clean.. + git clean -xffd + git reset -q --hard + git reset -q --hard "${repo_submodule_rev_wcomponent_wstatus}" + git clean -xffd + else + # we do not have the 'content' tag available - investigate its root + cd $(dirname $0) + ./ccm-baseline-history-get-root.sh "${ccm_submodule4part})" + exit 1 + fi + + if [[ ${push_tags_in_submodules} == "true" ]]; then + git tag -f -a -m "Please see tag in master repo for info: ${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" "${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" + git push ${git_remote_submodule_to_use} --recurse-submodules=no -f "${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" + fi + + cd ${root_dir} + ;; + *) + echo "[Submodule-mode] WHY are we here: submodule_update_mode is: ${submodule_update_mode} " + exit 1 + esac + + unset repo_submodule_rev + unset repo_submodule_rev_wcomponent_wstatus + unset ccm_submodule_inst + unset repo_submodule + + done + unset IFS + cd ${root_dir} + git add -A . > /dev/null 2>&1 + + if [[ ! ${repo_submodules} == "" ]]; then + if [[ -f .gitmodules ]]; then + cat .gitmodules + git add .gitmodules + else + echo "INFO: No .gitmodules initialized" + fi + fi + + git_set_execute_bit_in_index_of_extensions + git_set_execute_bit_in_index_of_unix_tool_file_executable + + export GIT_COMMITTER_DATE=$(git log -1 --format='%cd' ${repo_convert_rev_tag}) && [[ -z ${GIT_COMMITTER_DATE} ]] && return 1 + export GIT_COMMITTER_NAME=$(git log -1 --format='%cn' ${repo_convert_rev_tag} ) && [[ -z ${GIT_COMMITTER_NAME} ]] && return 1 + export GIT_COMMITTER_EMAIL=$(git log -1 --format='%ce' ${repo_convert_rev_tag} ) && [[ -z ${GIT_COMMITTER_EMAIL} ]] && return 1 + + export GIT_AUTHOR_DATE=$(git log -1 --format='%ad' ${repo_convert_rev_tag} ) && [[ -z ${GIT_AUTHOR_DATE} ]] && return 1 + export GIT_AUTHOR_NAME=$(git log -1 --format='%an' ${repo_convert_rev_tag} ) && [[ -z ${GIT_AUTHOR_NAME} ]] && return 1 + export GIT_AUTHOR_EMAIL=$(git log -1 --format='%ae' ${repo_convert_rev_tag} ) && [[ -z ${GIT_AUTHOR_EMAIL} ]] && return 1 + + echo "git commit content of ${repo_convert_rev_tag}" + git commit -C ${repo_convert_rev_tag} --reset-author || ( echo "Empty commit.." ) + + git submodule status || { + exit_code=$? + cat .gitmodules + exit $exit_code + } + git status + + set +x + echo "#####################" + echo "# git sizes:" + echo "#####################" + [[ -d .git/lfs ]] && printf "INFO: size: %s %s\n" $(du -sh .git/lfs) + [[ -d .git/lfs ]] && printf "INFO: LFS files count in this commit: %s\n" $( git lfs ls-files HEAD | wc -l ) + [[ -d .git/objects ]] && printf "INFO: size: %s %s\n" $(du -sh .git/objects) + [[ -d .git/modules ]] && printf "INFO: size: %s %s\n" $(du -sh .git/modules) + echo "#####################" + [[ ${debug:-} == "true" ]] && set -x + + # reset the committer to get the correct set for the commiting the tag. There is no author of the tag + export GIT_AUTHOR_DATE=$(git tag -l --format="%(taggerdate:iso8601)" "${repo_convert_rev_tag}" | awk -F" " '{print $1 " " $2}') && [[ -z ${GIT_AUTHOR_DATE} ]] && return 1 + export GIT_COMMITTER_DATE=${GIT_AUTHOR_DATE} + export GIT_COMMITTER_NAME=$(git tag -l --format="%(taggername)" "${repo_convert_rev_tag}" ) && [[ -z ${GIT_COMMITTER_NAME} ]] && return 1 + export GIT_COMMITTER_EMAIL=$(git tag -l --format="%(taggeremail)" "${repo_convert_rev_tag}" ) && [[ -z ${GIT_COMMITTER_EMAIL} ]] && return 1 + + echo "Get tag content of: ${repo_convert_rev_tag}" + git tag -l --format '%(contents)' "${repo_convert_rev_tag}" > ./tag_meta_data.txt + echo "git commit content of ${repo_convert_rev_tag}" + echo "git tag ${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized} based on ${repo_convert_rev_tag}" + git tag -a -F ./tag_meta_data.txt "${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" + rm -f ./tag_meta_data.txt + + # Do not consider submodules + if [[ ${push_to_remote_during_conversion:-} == "true" ]]; then + echo "INFO: Configured to push to remote: git push ${git_remote_to_use} --recurse-submodules=no -f ${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" + git push ${git_remote_to_use} --recurse-submodules=no -f "${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" || { + if [[ ${https_remote_common:-} != "" ]]; then + [[ -d .git/lfs ]] && { + git lfs fetch ${https_remote_common} --all + git lfs fetch ${git_remote_to_use} --all + } + git push ${git_remote_to_use} --recurse-submodules=no -f "${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" + else + echo "ERROR: something when wrong in pushing... and it is not due to use of common repo" + fi + } + if [[ ${https_remote_common:-} != "" ]]; then + git push ${https_remote_common} --recurse-submodules=no -f "${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" || { + [[ -d .git/lfs ]] && { + git lfs fetch ${https_remote_common} --all + git lfs fetch ${git_remote_to_use} --all + git push ${git_remote_submodule_to_use} --recurse-submodules=no -f "${repo_convert_rev_tag_wcomponent_wstatus_gitnormalized}" + } + } + fi + else + echo "INFO: Skip push to remote" + fi + + set +x + echo "============================================================================" + echo " DONE: $repo_convert_rev_tag_wcomponent_wstatus" + echo "============================================================================" + [[ ${debug:-} == "true" ]] && set -x + + unset GIT_AUTHOR_DATE + unset GIT_AUTHOR_NAME + unset GIT_AUTHOR_EMAIL + unset GIT_COMMITTER_DATE + unset GIT_COMMITTER_NAME + unset GIT_COMMITTER_EMAIL + unset root_dir + unset tag_to_convert + unset repo_convert_rev_tag + unset repo_convert_rev_tag_wcomponent_wstatus + unset repo_convert_rev_tag_wcomponent_wstatus_gitnormalized + unset repo_baseline_rev_tag_wcomponent_wstatus + unset repo_baseline_rev_tag_wcomponent_wstatus_lookup + unset repo_baseline_rev_tag_wcomponent_wstatus_gitnormalized + unset ccm_repo_convert_rev_tag + # From subfunction + unset proj_name + unset proj_version + unset proj_instance + unset project_release + unset ccm_baseline_obj_and_status_release_this + unset ccm_baseline_obj + unset ccm_baseline_status + unset ccm_baseline_release + +} + +function reset_converted_tags_except_init_remote_n_local() { + echo "Delete all local and remote tags '^${repo_name}/.*/.*_[dprtis][eueenq][lblsta]$'" + git tag | grep "^${repo_name}/.*/.*_[dprtis][eueenq][lblsta]$" | xargs --no-run-if-empty git push ${git_remote_to_use} --delete || echo "Some tags might not be on the remote - never mind" + git tag | grep "^${repo_name}/.*/.*_[dprtis][eueenq][lblsta]$" | xargs --no-run-if-empty git tag --delete || echo "Some tags might not be on the remote - never mind" +} + +function reset_converted_init_tag_remote_n_local() { + echo "Delete local and remote tag ${repo_name}/${repo_init_tag}/${repo_init_tag}" + git tag | grep "^${repo_name}/${repo_init_tag}/${repo_init_tag}$" | xargs --no-run-if-empty git push ${git_remote_to_use} --delete || echo "Some tags might not be on the remote - never mind" + git tag | grep "^${repo_name}/${repo_init_tag}/${repo_init_tag}$" | xargs --no-run-if-empty git tag --delete || echo "Some tags might not be on the remote - never mind" +} + +lock_repo_init_file="${execution_root_directory}/repo_under_construction_lock.txt" + +if [[ -f ${lock_repo_init_file} ]]; then + echo "INFO: Init construction of repo $repo_name did not complete - restart process" + rm -rf $repo_name +fi + +if [[ "${execute_mode}" == "reclone" ]]; then + echo "INFO: execute_mode is: '${execute_mode}'" + rm -rf ${repo_name} +fi + +if [[ ! -d "${repo_name}" ]] ; then + #initialize repo + echo "LOCK Repo: ${repo_name} cloned from: ${git_remote_to_use} is under init construction" > ${lock_repo_init_file} + git clone ${git_remote_to_use} + cd ${repo_name} + git fetch ${git_remote_to_use_orig} --tags --force +refs/heads/*:refs/remotes/origin/* + git branch -a + git tag + git reset -q --hard ${repo_init_tag} + git clean -xffd + + reset_converted_tags_except_init_remote_n_local + reset_converted_init_tag_remote_n_local + + export GIT_AUTHOR_DATE=$(git tag -l --format="%(taggerdate:iso8601)" ${repo_init_tag} | awk -F" " '{print $1 " " $2}') + export GIT_COMMITTER_DATE=${GIT_AUTHOR_DATE} + + for gitignore_path_n_file in $(echo ${gitignore_path_n_files} | sed -e 's/:/ /g'); do + gitignore_file_name=$(echo ${gitignore_path_n_file} | cut -d "@" -f 1) + gitignore_rel_path=$(echo ${gitignore_path_n_file} | cut -d "@" -f 2) + gitignore_full_path_name="${execution_root_directory}/${gitignore_file_name}" + if [[ ! -f ${gitignore_full_path_name} ]]; then + echo "${gitignore_full_path_name} does not exist.. Current dir:" + pwd + echo " .. Consider full path.." + exit 1 + else + mkdir -p ${gitignore_rel_path} + cp ${gitignore_full_path_name} ${gitignore_rel_path}/.gitignore + fi + done + + git_initialize_lfs_n_settings + for gitattributes_path_n_file in $(echo ${gitattributes_path_n_files} | sed -e 's/:/ /g'); do + gitattributes_file_name=$(echo ${gitattributes_path_n_file} | cut -d "@" -f 1) + gitattributes_rel_path=$(echo ${gitattributes_path_n_file} | cut -d "@" -f 2) + gitattributes_full_path_name="${execution_root_directory}/${gitattributes_file_name}" + if [[ ! -f ${gitattributes_full_path_name} ]]; then + echo "${gitattributes_full_path_name} does not exist.. - skip" + else + mkdir -p ${gitattributes_rel_path} + cp ${gitattributes_full_path_name} ${gitattributes_rel_path}/.gitattributes + fi + done + + git add -A . + git status + + echo "git commit init : ${repo_name}/${repo_init_tag}/${repo_init_tag}" + git commit -C "$repo_init_tag" --amend --reset-author + echo "git tag init commit: ${repo_name}/${repo_init_tag}/${repo_init_tag}" + git tag -f -a -m $(git tag -l --format '%(contents)' ${repo_init_tag}) ${repo_name}/${repo_init_tag}/${repo_init_tag} + + unset GIT_AUTHOR_DATE + unset GIT_COMMITTER_DATE + + git reset -q --hard ${repo_name}/${repo_init_tag}/${repo_init_tag} + git clean -xffd + + git ls-tree -r ${repo_name}/${repo_init_tag}/${repo_init_tag} + + if [[ ${push_to_remote_during_conversion:-} == "true" ]]; then + echo "INFO: Configured to push to remote: git push ${git_remote_to_use} --recurse-submodules=no -f ${repo_name}/${repo_init_tag}/${repo_init_tag}" + git push ${git_remote_to_use} --recurse-submodules=no -f ${repo_name}/${repo_init_tag}/${repo_init_tag} + else + echo "INFO: Skip push to remote: ${git_remote_to_use}" + fi + + pwd # we are still in the root repo + echo "UNLOCK repo: ${repo_name} as init construction completed.." && rm -f ${lock_repo_init_file} +else + echo "Already cloned and initialized" + cd ${repo_name} + if [[ "${execute_mode}" == "normal" ]]; then + echo "INFO: execute_mode is: '${execute_mode}'" + echo "Reset local tags in scope '^${repo_name}/.*/.*_[dprtis][eueenq][lblsta]$' and then start from begin of '^${repo_name}/init/init$'" + if [[ ! $( git tag | grep -v "^${repo_name}/init/init$" | grep "^${repo_name}/.*/.*_[dprtis][eueenq][lblsta]$" | xargs --no-run-if-empty git tag --delete ) ]] ; then + echo "No tags found" + fi + git fetch ${git_remote_to_use} --tags --force +refs/heads/*:refs/remotes/origin/* + git fetch ${git_remote_to_use} -ap +refs/heads/*:refs/remotes/origin/* + elif [[ "${execute_mode}" == "continue_locally" ]];then + echo "INFO: execute_mode is: '${execute_mode}'" + echo "Do not delete already converted tags and fetch again - just continue in workspace as is" + elif [[ "${execute_mode}" == "reset_remote_n_local" ]];then + echo "INFO: execute_mode is: '${execute_mode}'" + git fetch ${git_remote_to_use} --tags --force +refs/heads/*:refs/remotes/origin/* + git fetch ${git_remote_to_use} -ap +refs/heads/*:refs/remotes/origin/* + reset_converted_tags_except_init_remote_n_local + fi + git_initialize_lfs_n_settings +fi + +for sha1 in $(git log --topo-order --oneline --all --pretty=format:"%H " | tac) ; do + echo "Processing: $sha1" + tags=$(git tag --points-at "${sha1}" | grep -v "^${repo_name}/init/init$" | grep -v "^.*/.*/.*_[dprtis][eueenq][lblsta]$" | grep -v '/' | grep "^.*_[dprtis][eueenq][lblsta]$" || echo "") + if [[ "${tags}" == "" ]]; then + converted_tags=$(git tag --points-at "${sha1}" | grep .*/.*/.*_[dprtis][eueenq][lblsta]$ || echo "") + echo "INFO : No unconverted tags found - These are the new tags found - list and continue" + echo "${converted_tags}" + continue + fi + for project_revision in $(git tag --points-at "${sha1}" | grep -v "^${repo_name}/init/init$" | grep -v "^.*/.*/.*_[dprtis][eueenq][lblsta]$" | grep "^.*_[dprtis][eueenq][lblsta]$" || echo "@@@" ); do + [[ "${repo_name}/${repo_init_tag}/${repo_init_tag}" == "${project_revision}" ]] && continue + [[ "${repo_init_tag}" == "${project_revision}" ]] && continue + [[ "@@@" == "${project_revision}" ]] && continue + convert_revision ${project_revision} + done + echo "Done: $sha1" +done + +[[ -d .git/lfs ]] && printf "Git LFS in total: %s\n" $( git lfs ls-files --all | wc -l ) +[[ -d .git/lfs ]] && echo "Store list of LFS files in: ${execution_root_directory}/git_lfs_files.txt" +[[ -d .git/lfs ]] && git lfs ls-files --all > ${execution_root_directory}/git_lfs_files.txt + + diff --git a/src/main/resources/toGit/migration/sources/ccm/ccm-extract-baseline-project-metadata.sh b/src/main/resources/toGit/migration/sources/ccm/ccm-extract-baseline-project-metadata.sh new file mode 100644 index 0000000..620cf65 --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/ccm-extract-baseline-project-metadata.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash +set -e +set -u + +[[ "${debug:-}" == "true" ]] && set -x + +# Load functions +source $(dirname $0)/_ccm-functions.sh || source ./_ccm-functions.sh + +[[ -z $1 ]] && ( echo "Please set parameter 1 to ccm 4 part project name - exit 1" >&2 && exit 1 ) +ccm_4name=$1 + +[[ "${ccm_4name}" =~ ${regex_ccm4part} ]] || { + echo "$0: 4part does not comply" + exit 1 + } +ccm_project_name=${BASH_REMATCH[1]} +repo_convert_rev_tag=${BASH_REMATCH[2]} +repo_convert_type=${BASH_REMATCH[3]} +repo_convert_instance=${BASH_REMATCH[4]} + + +[[ -z ${2:-} ]] && ( echo "Please set parameter 2 to Jira Project Key - exit 1" >&2 && exit 1 ) +jira_project_key=$2 + +[[ -z ${3:-} ]] && ( echo "Please set parameter 3 to 'commit' or 'tag' - exit 1" >&2 && exit 1 ) +target_type=$3 + +jira_task_to_jira_issue_base=9000000 + +if [[ $target_type == "tag" ]] ; then + extract_data_epic_level="false" + extract_data_story_level="true" + extract_data_ccm_task_level="false" + extract_data_ccm_task_handle_dirtytasks_separately="false" + extract_data_ccm_task_verbosed_level="false" +elif [[ $target_type == "commit" ]] ; then + extract_data_epic_level="false" + extract_data_story_level="true" + extract_data_ccm_task_level="true" + extract_data_ccm_task_handle_dirtytasks_separately="false" + extract_data_ccm_task_verbosed_level="false" +else + echo "Parameter 5 is not set to 'commit' or 'tag' - exit 1" >&2 + exit 1 +fi + +# FIXME: This is a bit of a hack to determine the database type - should be done as a parameter/config as the database config +ccm_current_db=$(ccm status -f "%database %current_session" | grep TRUE | awk -F " " '{print $1}') +case ${ccm_current_db} in + /data/ccmdb/db_x|/data/ccmdb/db_y) + epic_level_header="Change Requests: (CR)" + epic_level_release_attr="TargetRelease" + epic_level_epic2story_relation="associatedWP" + + story_level_header="Work Packages (WP)" + story_level_release_attr="TargetRelease" + require_baseline_object="false" + ;; + /data/ccmdb/z) + epic_level_header="Master Change Requests: (MCR)" + epic_level_release_attr="release" + epic_level_epic2story_relation="associatedImpl" + + story_level_header="Implementation Change Requests(ICR)" + story_level_release_attr="release" + require_baseline_object="false" + ;; + *) + epic_level_header="Master Change Requests: (MCR)" + epic_level_release_attr="release" + epic_level_epic2story_relation="associatedImpl" + story_level_header="Implementation Change Requests" + story_level_release_attr="release" + require_baseline_object="false" + ;; +esac + +test "${ccm_project_name}x" == "x" && ( echo "'ccm_project_name' not set - exit" >&2 && exit 1 ) +test "${repo_convert_rev_tag}x" == "x" && ( echo "'repo_convert_rev_tag' not set - exit" >&2 && exit 1 ) +test "${repo_convert_instance}x" == "x" && ( echo "'repo_convert_rev_tag' not set - exit" >&2 && exit 1 ) + +output_file="./meta_data.txt" +rm -f ${output_file} + +function handle_task_attrs { + local _task_number_attrs=$1 + if [[ ${_task_number_attrs} == "none" ]] ; then + echo "">> ${output_file} + else + # 1 2 3 4 5 6 + regex='^(.+)@@@(.+)@@@(.+)@@@(.+)@@@(.+)@@@(.+)$' + [[ ${_task_number_attrs} =~ $regex ]] || exit 1 + task_objectname=${BASH_REMATCH[1]} + task_number=${BASH_REMATCH[2]} + task_create_time=${BASH_REMATCH[3]} + task_resolver=${BASH_REMATCH[4]} + task_status=${BASH_REMATCH[5]} + task_release="${BASH_REMATCH[6]}" + task_synopsis=$(ccm attr -show task_synopsis ${task_objectname}) + jira_subtask_issue_number=$(($jira_task_to_jira_issue_base + $task_number)) + printf "%-5s %-15s %-10s %-6s %-9s %-30s %s\n" " $loop_number)" "${jira_project_key}-${jira_subtask_issue_number}" "$task_create_time" "$task_resolver" "$task_status" "$task_release" "$task_synopsis" >> ${output_file} + fi +} +exit_code="0" +find_n_set_baseline_obj_attrs_from_project "${ccm_project_name}${ccm_delim}${repo_convert_rev_tag}:project:${repo_convert_instance}" "verbose_false" || exit_code=$? +if [[ "${exit_code}" != "0" ]] ; then + echo "ERROR: Project not found: ${ccm_project_name}${ccm_delim}${repo_convert_rev_tag}:project:${repo_convert_instance}" >&2 + exit ${exit_code} +fi + +if [[ "${ccm_baseline_obj:-}" != "" ]]; then + objectname="${ccm_project_name}${ccm_delim}${repo_convert_rev_tag}:project:${repo_convert_instance}" + printf "Project: ${objectname} <-> Baseline object: ${ccm_baseline_obj}\n\n" >> ${output_file} + ccm baseline -show info "${ccm_baseline_obj}" -f " Build: %build\n Description: %description\n Release: %release\n Purpose: %purpose\n" >> ${output_file} + + ccm baseline -show projects "${ccm_baseline_obj}" -f "%objectname %release %owner %{create_time[dateformat='yyyy-MM-dd HH:MM:SS']} Baseline: -> %baseline" >> ${output_file} + + printf "\nAll baseline objects related to project: ${ccm_project_name}${ccm_delim}${repo_convert_rev_tag}:project:${repo_convert_instance}\n" >> ${output_file} + ccm query "has_project_in_baseline('${objectname}')" \ + -sby create_time -f "%objectname %release %create_time" >> ${output_file} + echo >> ${output_file} + + if [[ $extract_data_epic_level == "true" ]]; then + echo >> ${output_file} + echo "${epic_level_header}:" >> ${output_file} + ccm query "has_${epic_level_epic2story_relation}(has_associated_task((is_task_in_baseline_of('${ccm_baseline_obj}') or is_dirty_task_in_baseline_of('${ccm_baseline_obj}'))))" -f "${jira_project_key}-%problem_number %resolver %${epic_level_release_attr} %problem_synopsis" >> ${output_file} || echo "" >> ${output_file} + fi + + if [[ $extract_data_story_level == "true" ]]; then + echo >> ${output_file} + echo "Fully integrated ${story_level_header}:" >> ${output_file} + ccm baseline -show fully_included_change_requests -groupby "${epic_level_release_attr}: %${epic_level_release_attr}" -f "${jira_project_key}-%problem_number %resolver %${epic_level_release_attr} %problem_synopsis" "${ccm_baseline_obj}" >> ${output_file} || echo "" >> ${output_file} + + echo >> ${output_file} + echo "Partially integrated ${story_level_header}:" >> ${output_file} + ccm baseline -show partially_included_change_requests -groupby "${epic_level_release_attr}: %${epic_level_release_attr}" -f "${jira_project_key}-%problem_number %resolver %${epic_level_release_attr} %problem_synopsis" "${ccm_baseline_obj}" >> ${output_file} || echo "" >> ${output_file} + fi + + if [[ $extract_data_ccm_task_level == "true" ]]; then + echo >> ${output_file} + if [[ ${extract_data_ccm_task_handle_dirtytasks_separately} == "true" ]]; then + echo "Tasks integrated in baseline:" >> ${output_file} + query1="is_task_in_baseline_of('${ccm_baseline_obj}')" + query2="is_dirtytask_in_baseline_of('${ccm_baseline_obj}')" + else + echo "All tasks integrated in baseline:" >> ${output_file} + query1="is_task_in_baseline_of('${ccm_baseline_obj}') or is_dirty_task_in_baseline_of('${ccm_baseline_obj}')" + fi + + IFS=$'\n\r' + loop_number=1 + for task_number_attrs in $(ccm query "${query1}" -u -f "%objectname@@@%task_number@@@%{create_time[dateformat='yyyy-MM-dd HH:MM:SS']}@@@%resolver@@@%status@@@%release" || ( [[ $? == 6 ]] && echo "none" ) ) ; do + handle_task_attrs "$task_number_attrs" + loop_number=$((loop_number + 1)) + done + unset IFS + + if [[ ${extract_data_ccm_task_handle_dirtytasks_separately} == "true" ]]; then + echo >> ${output_file} + echo "Dirty tasks integrated in baseline: ( listed in baseline, but has not effect as it's objects are behind the baseline project )" >> ${output_file} + IFS=$'\n\r' + loop_number=1 + for task_number_attrs in $(ccm query "${query2}" -u -f "%objectname@@@%task_number@@@%{create_time[dateformat='yyyy-MM-dd HH:MM:SS']}@@@%resolver@@@%status@@@%release" || ( [[ $? == 6 ]] && echo "none" ) ) ; do + handle_task_attrs "$task_number_attrs" + loop_number=$((loop_number + 1)) + done + unset IFS + fi + if [[ $extract_data_ccm_task_verbosed_level == "true" ]]; then + echo >> ${output_file} + echo "Tasks integrated in baseline and/or project (verbosed):" >> ${output_file} + if [[ ${query2} == "" ]]; then + query3="${query1}" + else + query3="${query1} or ${query2}" + fi + + integrated_tasks=$(ccm query "${query3}" || ( [[ $? == 6 ]] && echo "none" )) + if [[ ${integrated_tasks} == "none" ]]; then + echo "" >> ${output_file} + else + ccm task -sh info -v @ >> ${output_file} + fi + echo >> ${output_file} + fi + fi + + +else + [[ "${require_baseline_object}" == "true" ]] && ( echo "ERROR: It is expected to have a baseline object due to configuration: require_baseline_object=true for this database: ${ccm_current_db}" >&2 && exit 2 ) + objectname="${ccm_project_name}${ccm_delim}${repo_convert_rev_tag}:project:${repo_convert_instance}" + + printf "Project: ${objectname} <-> Baseline object: NONE\n\n" >> ${output_file} + + echo "Project baseline:" >> ${output_file} + ccm query "is_baseline_project_of('${objectname}')" -f "%displayname" >> ${output_file} || echo " " >> ${output_file} + echo >> ${output_file} + + if [[ ${epic_level_header:-} != "" ]]; then + echo "${epic_level_header}:" >> ${output_file} + ccm query "has_${epic_level_epic2story_relation}(has_associated_task(is_task_in_folder_of(is_folder_in_rp_of('${objectname}'))))" -f "${jira_project_key}-%problem_number %resolver %release %problem_synopsis" >> ${output_file} || echo "" >> ${output_file} + + echo >> ${output_file} + fi + + echo "Related/Integrated ${story_level_header}:" >> ${output_file} + ccm query "has_associated_task(is_task_in_folder_of(is_folder_in_rp_of('${objectname}')))" -f "${jira_project_key}-%problem_number%resolver %release %problem_synopsis" >> ${output_file} || echo "" >> ${output_file} + + echo >> ${output_file} + echo "Tasks integrated in project:" >> ${output_file} + #ccm query "is_task_in_folder_of(is_folder_in_rp_of('${objectname}'))" -f "%displayname %{create_time[dateformat='yyyy-M-dd HH:MM:SS']} %resolver %status %release %task_synopsis" >> ${output_file} || echo "" >> ${output_file} + IFS=$'\n\r' + loop_number=1 + for task_number_attrs in $(ccm query "status!='task_automatic' and (is_task_in_folder_of(is_folder_in_rp_of('${objectname}')) or is_task_in_rp_of('${objectname}'))" -u -f "%objectname@@@%task_number@@@%{create_time[dateformat='yyyy-MM-dd HH:MM:SS']}@@@%resolver@@@%status@@@%release" | tail -n +2) ; do + handle_task_attrs "$task_number_attrs" + loop_number=$((loop_number + 1)) + done + unset IFS + + if [[ ${extract_data_ccm_task_verbosed_level:-} == "true" ]]; then + echo >> ${output_file} + echo "Tasks integrated in baseline and/or project (verbosed):" >> ${output_file} + integrated_tasks=$(ccm query "status!='task_automatic' and (is_task_in_folder_of(is_folder_in_rp_of('${objectname}')) or is_task_in_rp_of('${objectname}'))" || ( [[ $? == 6 ]] && echo "none" )) + if [[ ${integrated_tasks} == "none" ]]; then + echo "" >> ${output_file} + else + ccm task -sh info -v @ >> ${output_file} + fi + echo >> ${output_file} + fi + +fi +cat ${output_file} +rm -f ${output_file} \ No newline at end of file diff --git a/src/main/resources/toGit/migration/sources/ccm/ccm-get-status-from-baseline-or-project.sh b/src/main/resources/toGit/migration/sources/ccm/ccm-get-status-from-baseline-or-project.sh new file mode 100644 index 0000000..aad1d38 --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/ccm-get-status-from-baseline-or-project.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -u +set -e + +[[ "${debug:-}" == "true" ]] && set -x + +# Load functions +source $(dirname $0)/_ccm-functions.sh || source ./_ccm-functions.sh + +ccm_proj_obj_string="$1" + +exit_code="0" +find_n_set_baseline_obj_attrs_from_project "${ccm_proj_obj_string}" "verbose_false" || exit_code=$? +if [[ "${exit_code}" != "0" ]] ; then + echo "ERROR: Project not found: ${ccm_proj_obj_string}" + exit ${exit_code} +fi + +if [[ "${ccm_baseline_status:-}" == "" ]]; then + # We could not set status from baseline object - take it from the project + ccm_baseline_status=$(ccm attr -show status "${ccm_proj_obj_string}" | sed -e 's/ //g' | cut -c1-3) +else + ccm_baseline_status=$(echo ${ccm_baseline_status} | cut -c1-3) +fi +if [[ "${ccm_baseline_status:-}" == "" ]] ; then + echo "Something went wrong as no status is set" + exit 1 +else + echo ${ccm_baseline_status} +fi \ No newline at end of file diff --git a/src/main/resources/toGit/migration/sources/ccm/ccm-get-translated_string.sh b/src/main/resources/toGit/migration/sources/ccm/ccm-get-translated_string.sh new file mode 100644 index 0000000..e7aacc5 --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/ccm-get-translated_string.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -u +set -e + +[[ "${debug:-}" == "true" ]] && set -x + +# Load functions +source $(dirname $0)/_ccm-functions.sh || source ./_ccm-functions.sh + +byref_function=$1 +shift +[[ "${byref_function:-}" == "" ]] && ( echo "'byref_function' not set as parameter 1 - exit" && exit 1 ) + +result="" +case $byref_function in + byref_translate_from_ccm_name2git_repo) + ccm_project_name=$1 + eval $byref_function "\"${ccm_project_name}\"" result + ;; + *) + eval $byref_function $@ result + ;; +esac + +printf "$result" diff --git a/src/main/resources/toGit/migration/sources/ccm/ccm-get_project_rev_count.sh b/src/main/resources/toGit/migration/sources/ccm/ccm-get_project_rev_count.sh new file mode 100644 index 0000000..4013e88 --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/ccm-get_project_rev_count.sh @@ -0,0 +1,12 @@ +#!/bin/bash --login +set -e +set -u +[[ ${debug:-} == true ]] && set -x + +IFS=$'\r\n' +{ + for proj in $(ccm query "type='project' and (status='integrate' or status='released') " -u -f "%name" | sort -u ) ; do + rev_count=$(ccm query "type='project' and name='${proj}' and ( status='released' or status='integrate' and status='test' and status='sqa')" -u -f "%objectname" | wc -l ) + printf "%s : %s\n" "$rev_count" "$proj" + done +} | sort -rh \ No newline at end of file diff --git a/src/main/resources/toGit/migration/sources/ccm/ccm.user.properties b/src/main/resources/toGit/migration/sources/ccm/ccm.user.properties new file mode 100644 index 0000000..ebfb41d --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/ccm.user.properties @@ -0,0 +1 @@ +ccm.cli.format.date=yyyy-MM-dd HH\:MM\:SS \ No newline at end of file diff --git a/src/main/resources/toGit/migration/sources/ccm/ccm_find_dangeling_tasks.sh b/src/main/resources/toGit/migration/sources/ccm/ccm_find_dangeling_tasks.sh new file mode 100644 index 0000000..bcc5539 --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/ccm_find_dangeling_tasks.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash + +set -euo pipefail + +[[ ${debug:-} == true ]] && set -x + + +#Load DB settings +source ${BASH_SOURCE%/*}/${ccm_db}_settings.sh 2> /dev/null || source ./${ccm_db}_settings.sh + +if [[ ${type:-} == "task_assigned" || ${1:-} == "task_assigned" ]]; then + query="type='task' and release match '*' and status='task_assigned'" + time_date_field="create_time" + subdir=task_assigned + [[ ${jira_issue_state:-} == "" ]] && jira_issue_state="TO DO" + echo "Running in task_assigned mode" +else + query="type='task' and release match '*' and status='completed' and ( is_no_task_in_baseline() ) " + time_date_field="completion_date" + subdir=completed + [[ ${jira_issue_state:-} == "" ]] && jira_issue_state="IN VALIDATION" + echo "Running in task completed mode" +fi + +[[ ${objects_max:-} == "" ]] && objects_max=50 + +echo $query +ccm query "$query" -u -f %objectname | wc -l + +mkdir -p ${subdir} +rm -rf ${subdir}/dirty.txt + +function investigate_projects { + for project in $( ccm finduse ${task_object} -all_projs | sed -e 's/\t\t//' ); do + project_status=$(ccm attr -show status $project) + if [[ $project_status == "integrate" || $project_status == "released" || project_status == 'sqa' || project_status == "test" ]]; then + project_found=true + echo "ERROR : detect conflicts for the projects due to 'folder_in_rp' can be updated after checkin" + exit 1 + printf "OK: $task_number $project $project_status\n" >> ${subdir}/overview.txt + printf "OK: %s : %s : %s : %s : %s : %s\n" \ + "$task_number" \ + "$(ccm attr -show release ${task_object})" \ + "$(ccm attr -show status ${task_object})" \ + "$(ccm attr -show resolver ${task_object})" \ + "$(ccm attr -show ${time_date_field} ${task_object})" \ + "$(ccm attr -show task_synopsis ${task_object})" >> ${subdir}/overview.txt + + ccm finduse ${task_object} -all_projs >> ${subdir}/in_static_project/$task + return + fi + done +} + +#yyyy-MM-dd HH:mm:ss + +IFS=$'\r\n' +for task_number_resolver in $(ccm query "$query" -u -f "%task_number@%resolver" ) ; do + task_number=${task_number_resolver%@*} + task_resolver=${task_number_resolver#*@} + temp_dir_path=${subdir}/_${task_number}-${task_resolver} + final_dir_path=${subdir}/${task_number}-${task_resolver} + + if [[ -d ${final_dir_path} ]]; then + printf "|" + continue + else + printf '\n' + fi + rm -rf ${temp_dir_path} + rm -f ${temp_dir_path}/*.commit ${temp_dir_path}/*.csv + + task_object=$(ccm query "task('$task_number')" -u -f %objectname) + + + baseline_found=false + # sleep 1 + printf "${task_number}: " + for baseline in $(ccm query "has_dirty_task_in_baseline('${task_object}') or has_task_in_baseline('${task_object}') " || true ) ; do + baseline_found=true + [[ ${details:-} == true ]] && echo $baseline + done + + if [[ ${baseline_found:-} == false ]] ; then + project_found=false + #investigate_projects + if [[ ${project_found} == false ]]; then + object_count="$(ccm query "is_associated_cv_of('$task_object') and not (type='dir' or type='project')" -u -f "%objectname" | wc -l)" || exit_code=$? + if [[ $object_count -eq 0 ]]; then + printf " $object_count - skip" + continue + fi + mkdir -p ${temp_dir_path} + printf " $object_count " + printf "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n" \ + "\"${task_number}\"" \ + "\"${jira_issue_type}\"" \ + "\"${jira_issue_state}\"" \ + "\"${jira_issue_epic_state:-New}\"" \ + "\"$(ccm attr -show release ${task_object})\"" \ + "\"${task_resolver}\"" \ + "\"${task_resolver}\"" \ + "\"$(ccm attr -show ${time_date_field} ${task_object})\"" \ + "\"$(ccm attr -show ${time_date_field} ${task_object})\"" \ + "\"${object_count}\"" \ + "\"${task_number}-dangTask [ccmTask]: $(ccm attr -show task_synopsis ${task_object})\"" \ + "\"${task_number}-dangTask [ccmTask]: $(ccm attr -show task_synopsis ${task_object})\"" \ + "\"Details: ${details_link}/${final_dir_path}\"" \ + "\"${subdir}\"" > ${temp_dir_path}/task_info_${task_number}.csv + printf "%s-%s : %s\n" \ + "${jira_project_key}" \ + "${task_number}" \ + "${task_number}-dangTask [ccmTask]: $(ccm attr -show task_synopsis ${task_object})" \ + > ${temp_dir_path}/${task_number}.commit + ccm task -show info -v $task_number > ${temp_dir_path}/task_info_${task_number}.txt + if [[ ${folders:-} == true ]]; then + folders_found=false + { + printf "\n\nFolders:\n" + for folder in $( ccm finduse ${task_object} -all_folders | grep -A 100 Folder | sed -e 's/\t\t//' ); do + folders_found=true + done + echo + } >> ${temp_dir_path}/task_info_${task_number}.txt + fi + if [[ ${objects:-} == true ]]; then + echo >> ${temp_dir_path}/task_info_${task_number}.txt + printf "\n\nObjects and predecessors:\n" >> ${temp_dir_path}/task_info_${task_number}.txt + + if [[ $object_count -gt ${objects_max} ]]; then + printf "over 50 objects - skipped" + echo " - over 50 objects: $object_count - skipped" >> ${temp_dir_path}/task_info_${task_number}.txt + else + for object in $(ccm query "is_associated_cv_of('$task_object') and not (type='dir' or type='project')" -u -f "%objectname" ); do + echo "+ $object" >> ${temp_dir_path}/task_info_${task_number}.txt + ccm finduse -all_projs $object >> ${temp_dir_path}/task_info_${task_number}.txt + ccm query "has_successor('${object}')" -u -f " - predecessor: %objectname" >> ${temp_dir_path}/task_info_${task_number}.txt || { + exit_code=$? + [[ $exit_code != 6 ]] && { echo "ERROR: something is not right in predecessor of $task_number - $object"; exit $exit_code ; } + } + if [[ -e ${temp_dir_path}/${object} ]]; then + printf "." + else + printf "," + ccm cat $object > ${temp_dir_path}/_${object} + mv ${temp_dir_path}/_${object} ${temp_dir_path}/${object} + fi + done + fi + echo >> ${temp_dir_path}/task_info_${task_number}.txt + fi + mv ${temp_dir_path} ${final_dir_path} + fi + else + printf "WARNING: likely dirty" + printf "%s - skip - likely dirty\n" "$task_number" >> ${subdir}/dirty.txt + fi +done +echo + +echo "Collection csv for Jira" +echo "Issue id ,Issue Type ,Status, Epic Status, FixVersion,Assignee,Reporter,Date created,Date modified,CCM Objects, Summary,Epic Name,Description,Labels" > ${subdir}/all.csv + +find ${subdir} -name task_info_*.csv | xargs -I % cat % >> ${subdir}/all.csv +ls -la ${subdir}/all.csv + +echo "Collecting commit messages" +printf "${subdir}: Task information and objects\n\n" > ${subdir}/all.commit +find ${subdir} -name [[:digit:]]*.commit | xargs -I % cat % >> ${subdir}/all.commit +ls -la ${subdir}/all.commit +echo \ No newline at end of file diff --git a/src/main/resources/toGit/migration/sources/ccm/emptydir.gitignore b/src/main/resources/toGit/migration/sources/ccm/emptydir.gitignore new file mode 100644 index 0000000..407062b --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/emptydir.gitignore @@ -0,0 +1,4 @@ +# Ignore everything - added in CM/synergy to git migration as it was an empty directory +* +# except .gitignore +!.gitignore diff --git a/src/main/resources/toGit/migration/sources/ccm/execute-ccm-ccm2git.sh b/src/main/resources/toGit/migration/sources/ccm/execute-ccm-ccm2git.sh new file mode 100644 index 0000000..4dbd7d0 --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/execute-ccm-ccm2git.sh @@ -0,0 +1,206 @@ +#!/bin/bash --login +set -e +set -u + +# Load functions + +[[ ${ccm_db} == "" ]] && { + echo "ERROR: ccm_db is not set" + exit 1 +} +if [[ "${groovy_script:-}" == "" ]]; then + echo "groovy_script must be set" + exit 1 +else + export groovy_script=$(pwd)/${groovy_script} + [[ -e ${groovy_script} ]] || { echo "ERROR: does not exist: $groovy_script" && exit 1; } +fi +[[ "${git_user_name:-}" == "" ]] && { echo "ERROR: git_user_name env variable must be set. It is used for the committer and init commit author" && exit 1; } +[[ "${git_user_email:-}" == "" ]] && { echo "ERROR: git_user_email env variable must be set. It is used for the committer and init commit author" && exit 1; } +[[ "${git_email_domain:-}" == "" ]] && { echo "ERROR: git_email_domain env variable must be set. It is used for the author domain. The username part of email is retrieved from CM/Synergy" && exit 1; } +[[ "${git_server_path:-}" == "" ]] && { echo "ERROR: git_server_path env variable must be set" && exit 1; } +[[ "${jiraProjectKey:-}" == "" ]] && { echo "ERROR: jiraProjectKey env variable must be set" && exit 1; } +#[[ "${use_cached_project_list:-}" == "" ]] && { export use_cached_project_list="true" ; } +echo "git_user_name=${git_user_name}" +echo "git_user_email=${git_user_email}" +echo "git_email_domain=${git_email_domain}" +echo "git_server_path=${git_server_path}" +echo "jiraProjectKey=${jiraProjectKey}" + +if [[ "${my_workspace_root:-}" == "" ]]; then + echo "INFO: my_workspace_root is not set .. defaulting to $(pwd)" + export my_workspace_root=$(pwd) +else + echo "INFO: my_workspace_root=${my_workspace_root}" +fi +if [[ -e $my_workspace_root ]] ; then + if git -C $my_workspace_root rev-parse --git-dir 2> /dev/null ; then + echo "git rev-parse --git-dir show that my_workspace_root=${my_workspace_root} is inside a repo - not supported" + echo "If you want to store it inside a Jenkins workspace then consider to checkout repo to a subdir and use \${WORKSPACE} as my_workspace_root" + exit 1 + else + echo "INFO: All good .. my_workspace_root = $my_workspace_root exists, but is not inside a git repo" + fi +fi + +source "$(pwd)/${BASH_SOURCE%/*}/_ccm-start-stop-functions.sh" || source ./_ccm-start-stop-functions.sh +ccm-start ${ccm_db} +source "$(pwd)/${BASH_SOURCE%/*}/_ccm-functions.sh" || source ./_ccm-functions.sh + +[[ "${debug:-}" == "true" ]] && set -x + +whoami +env > env.env + +if ! which java ; then + if [[ ${JAVA_HOME:-} != "" ]]; then + export PATH="${JAVA_HOME}/bin:$PATH" + which java || { echo "ERROR: java not found in PATH nor via $JAVA_HOME/bin" ; exit 1; } + echo "INFO: prepending $JAVA_HOME/bin to PATH" + else + echo "ERROR: java not found in PATH. Please set it in PATH or set JAVA_HOME and it is set to JAVA_HOME/bin" + exit 1 + fi +fi +java -version + +trap ccm-stop EXIT + +[[ ${ccm_project_name:-} == "" ]] && { echo "ERROR: please set variable: ccm_project_name with option of [:]" ; exit 1; } +export ccm_project_name_wo_instance=$(echo ${ccm_project_name} | awk -F ":" '{print $1}') + +export ccm_project_instance=$(echo ${ccm_project_name} | awk -F ":" '{print $2}') +if [[ ${ccm_project_instance:-} == "" ]]; then + if [[ ${ccm_dcm_dbid:-} != "" ]]; then + ccm_project_instance="${ccm_dcm_dbid}#1" + else + ccm_project_instance=1 + fi +fi + +export ccm_project_name_orig="" + +byref_translate_from_git_repo2ccm_name $ccm_project_name_wo_instance $ccm_project_instance ccm_project_name_orig + +rm -rf git2git_params.env && touch git2git_params.env +if [[ "${wipe_repo_before:-}" == "true" ]] ; then + echo "Execute: wipe_repo_before" + rm -rf ${my_workspace_root}/${ccm_project_name_wo_instance}/repo + echo "execute_mode=reclone" >> git2git_params.env +else + echo "INFO: wipe_repo_before is empty [''|false|true]" +fi +if [[ "${wipe_checkout_workspace_before:-}" == "true" ]] ; then + echo "Execute: wipe_checkout_workspace_before" + rm -rf ${my_workspace_root}/${ccm_project_name_wo_instance}/ccm_wa +else + echo "INFO: wipe_checkout_workspace_before is empty [''|false|true]" +fi + +if [[ "${tag_to_be_removed:-}" != "" ]] ; then + echo "tag_to_be_removed=${tag_to_be_removed}" >> git2git_params.env + cd ${my_workspace_root}/${ccm_project_name_wo_instance}/repo/${ccm_project_name_wo_instance} + git push origin $tag_to_be_removed --delete || echo "Deleting failed - skip" + git push ssh://git@${git_server_path}/${ccm_project_name_wo_instance}_orig.git $tag_to_be_removed --delete || echo "Deleting failed - skip" + if [[ ${git_server_path_prod_ccm2git:-} != "" ]]; then + git push origin $tag_to_be_removed --delete || echo "Deleting failed - skip" + git push ssh://git@${git_server_path_prod_ccm2git}/${ccm_project_name_wo_instance}_orig.git $tag_to_be_removed --delete || echo "Deleting failed - skip" + fi + git tag --delete $tag_to_be_removed + cd ${WORKSPACE} +else + echo "INFO: tag_to_be_removed: is not set " +fi + +[[ ${ccm_proj_rev_exclusion_query:-} == "" ]] && { echo "INFO: ccm_proj_rev_exclusion_query is not set" ; } + +rm -rf project_baselines.txt && touch project_baselines.txt +echo "INFO: Finding the $ccm_project_name_orig revisions that has has_no_baseline_project" +ccm query "\ + type='project' \ + and name='${ccm_project_name_orig}' \ + and instance='${ccm_project_instance}' \ + and ( status='integrate' or status='test' or status='sqa' or status='released' ) \ + and has_no_baseline_project() ${ccm_proj_rev_exclusion_query:-}" \ + -u -f "%objectname" \ + >> project_baselines.txt || { + exit_code=$? + if [[ $exit_code -eq 6 ]]; then + echo "Empty result: has_no_baseline_project - never mind" + elif [[ $exit_code -ne 0 ]] ; then + echo "ERROR: has_no_baseline_project something went wrong - exit : $exit_code" + exit $exit_code + fi + } + echo "" >> project_baselines.txt +echo "INFO: Finding the $ccm_project_name_orig which is is_hist_leaf" +bash "$(pwd)/${BASH_SOURCE%/*}/ccm-baseline-history-get-root.sh" \ + "$( ccm query "\ + type='project' \ + and name='${ccm_project_name_orig}' \ + and instance='${ccm_project_instance}' \ + and ( status='integrate' or status='test' or status='sqa' or status='released' ) \ + and is_hist_leaf() " \ + -u -f "%objectname" | head -1 \ + )" >> project_baselines.txt || { + exit_code=$? + if [[ $exit_code -eq 6 ]]; then + echo "Empty result: is_hist_leaf - never mind" + elif [[ $exit_code -ne 0 ]] ; then + echo "ERROR: is_hist_leaf something went wrong - exit : $exit_code" + exit $exit_code + fi + } + echo "" >> project_baselines.txt + +if [[ -f project_baselines.txt ]]; then + ccm_project_baselines=$( sort -u < project_baselines.txt ) + export ccm_project_baselines +else + echo "No project revision found - Neither from 'no_baseline' nor from history traverse' - exit 10" + exit 10 +fi + +IFS=$'\r\n' +for ccm_project_baseline in $( sort -u < project_baselines.txt ) ; do + + [[ "${ccm_project_baseline}" =~ ${regex_ccm4part:?} ]] || { + echo "4part does not comply" + return 1 + } + revision=${BASH_REMATCH[2]} + + git_repo_4part="" + byref_translate_from_ccm_4part2git_repo_4part "${ccm_project_baseline}" git_repo_4part + + [[ -d ${my_workspace_root}/${ccm_project_name_wo_instance}/ccm_wa/ ]] && touch ${my_workspace_root}/${ccm_project_name_wo_instance}/ccm_wa/projects.txt + [[ -d ${my_workspace_root}/${ccm_project_name_wo_instance}/repo/ ]] && touch ${my_workspace_root}/${ccm_project_name_wo_instance}/repo/git_sizes.txt + + cd ${BASH_SOURCE%/*}/ + echo "calling: java -jar build/libs/*.jar in $(pwd)" + set -x + java -jar build/libs/*.jar \ + ${groovy_script} \ + start_project="${git_repo_4part}" \ + my_workspace_root=${my_workspace_root} \ + git_server_path=${git_server_path} \ + jiraProjectKey=${jiraProjectKey} + [[ "${debug:-}" == "true" ]] && set -x # restore debug setting + cd - + cp "${my_workspace_root}/${ccm_project_name_wo_instance}/ccm_wa/projects.txt" "${my_workspace_root}/${ccm_project_name_wo_instance}/ccm_wa/projects-${revision}.txt" + cp "${my_workspace_root}/${ccm_project_name_wo_instance}/ccm_wa/projects-${revision}.txt" "${WORKSPACE}/" +done +cat projects-*.txt | sort -u > projects.txt +for file in $(ls -1 -rt ${my_workspace_root}/${ccm_project_name_wo_instance}/repo/*@git_size.txt); do + size=$(cat $file | awk -F" " '{print $1}') + echo "${size} @@@ ${file}" >> git_sizes.txt +done +ccm_amount_of_versions=$( wc -l < git_sizes.txt ) +git_size=$(tail -1 git_sizes.txt | awk -F " " '{print $1}') + +{ + echo "ccm_amount_of_versions=${ccm_amount_of_versions}" + echo "git_size=${git_size}" + echo "ccm_project_name_wo_instance=${ccm_project_name_wo_instance}" + echo "ccm_project_instance=${ccm_project_instance}" +} > ccm.env diff --git a/src/main/resources/toGit/migration/sources/ccm/execute-ccm-git2git.sh b/src/main/resources/toGit/migration/sources/ccm/execute-ccm-git2git.sh new file mode 100644 index 0000000..06bbec7 --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/execute-ccm-git2git.sh @@ -0,0 +1,45 @@ +#!/bin/bash --login +set -x +set -u +set -e +set -o pipefail + +export ccm_project_name_wo_instance=$(echo ${ccm_project_name} | awk -F ":" '{print $1}') +export ccm_project_instance=$(echo ${ccm_project_name} | awk -F ":" '{print $2}') + +if [ "${ccm_project_instance}x" == "x" ]; then + export ccm_project_instance="1" +fi + +if [[ "${tag_to_be_removed:-}" != "" ]] ; then + cd ${ccm_project_name_wo_instance} + git push ssh://git@${git_server_path}/${ccm_project_name_wo_instance}.git $tag_to_be_removed --delete || echo "Deleting failed - skip" + git push ssh://git@${git_server_path}/${ccm_project_name_wo_instance}_orig.git $tag_to_be_removed --delete || echo "Deleting failed - skip" + if [[ ${git_server_path_prod_git2git:-} != "" ]] ; then + echo "Also handle production" + git push ssh://git@${git_server_path_prod_git2git}/${ccm_project_name_wo_instance}.git $tag_to_be_removed --delete || echo "Deleting failed - skip" + git push ssh://git@${git_server_path_prod_git2git}/${ccm_project_name_wo_instance}_orig.git $tag_to_be_removed --delete || echo "Deleting failed - skip" + fi + git tag --delete $tag_to_be_removed + cd ${WORKSPACE} +fi + +export PATH=${ccm_home_path}/bin:$PATH +export CCM_HOME=${ccm_home_path} + +which ccm +export CCM_ADDR=`ccm start -m -d /data/ccmdb/${ccm_db} -s ${ccm_server} -q` +[[ $CCM_ADDR == "" ]] && ( echo "CM/Synergy start failed" && exit 10 ) + +debug=true ${WORKSPACE}/ccm-convert-flat2submodules-existingRepoHistory.sh \ + "${ccm_project_name_wo_instance}" \ + "init" \ + "${repo_submodules}" \ + "${git_server_project}" \ + "${ccm_project_instance}" \ + "${gitignore_path_n_files:-}" \ + "${gitattributes_path_n_files:-}" + +echo "ccm_project_name_wo_instance=${ccm_project_name_wo_instance}" > ccm.env +echo "ccm_project_instance=${ccm_project_instance}" >> ccm.env + diff --git a/src/main/resources/toGit/migration/sources/ccm/git-fill-empty-dirs-with-gitignore.sh b/src/main/resources/toGit/migration/sources/ccm/git-fill-empty-dirs-with-gitignore.sh new file mode 100644 index 0000000..6e6a645 --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/git-fill-empty-dirs-with-gitignore.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -u +set -e +set -o pipefail +set -o posix + +[[ ${debug:-} == "true" ]] && set -x + +script_dir=$(dirname $(readlink -f $0)) +source $(dirname $0)/_git-functions.sh || source ./_git-functions.sh + +cd $1 +git_find_n_fill_empty_dirs_gitignore diff --git a/src/main/resources/toGit/migration/sources/ccm/git-find-n-move-tag.sh b/src/main/resources/toGit/migration/sources/ccm/git-find-n-move-tag.sh new file mode 100644 index 0000000..0b5a770 --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/git-find-n-move-tag.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +[[ ${debug:-} == "true" ]] && set -x +set -u +set -e +set -o pipefail + +HEAD_of_tree=HEAD + +for sha1 in $(git rev-list ${HEAD_of_tree} ) ; do + tag_source=$(git tag --points-at $sha1) || { echo $sha1: null ; continue ; } + echo $sha1: $tag_source +done +for sha1 in $( git rev-list ${HEAD_of_tree} ) ; do + tag_source=$(git tag --points-at $sha1) + commit_line=$(git show -s --format=@%cd@%s refs/tags/${tag_source}^{}) + + sha1_of_target=$(git log --format=%H@%cd@%s origin/master | grep -e "${commit_line}" | cut -d @ -f 1 ) || { echo "ERROR: $tag_source commit is not found on origin/master" ; exit 1; } + + if [[ $(wc -w <<< ${sha1_of_target} ) -gt 1 ]]; then + echo "ERROR: $sha1_of_target have more than one sha1" + fi + printf "%s %s\n" "${tag_source}" "${sha1_of_target}" + + tag_target=${tag_source} + tag_old_source="old/${tag_source}" + + # reset the committer to get the correct set for the commiting the tag. There is no author of the tag + export GIT_AUTHOR_DATE=$(git tag -l --format="%(taggerdate:iso8601)" "${tag_source}" | awk -F" " '{print $1 " " $2}') && [[ -z ${GIT_AUTHOR_DATE} ]] && return 1 + export GIT_COMMITTER_DATE=${GIT_AUTHOR_DATE} + export GIT_COMMITTER_NAME=$(git tag -l --format="%(taggername)" "${tag_source}" ) && [[ -z ${GIT_COMMITTER_NAME} ]] && return 1 + export GIT_COMMITTER_EMAIL=$(git tag -l --format="%(taggeremail)" "${tag_source}" ) && [[ -z ${GIT_COMMITTER_EMAIL} ]] && return 1 + + echo "Get tag content of: ${tag_source}" + git tag -l --format '%(contents)' "${tag_source}" > ./tag_meta_data.txt + + echo "Tag the source tag for history reasons: ${tag_source} -> ${tag_old_source}" + git tag -a -F ./tag_meta_data.txt "${tag_old_source}" "${tag_source}^{}" + + echo "content of ${tag_source} to ${sha1_of_target}" + echo "git tag ${tag_target} based on ${tag_source}" + git tag -a -F ./tag_meta_data.txt "${tag_target}" "${sha1_of_target}" -f + rm -f ./tag_meta_data.txt +done diff --git a/src/main/resources/toGit/migration/sources/ccm/git-remove-all-git-related-files-2plus-levels.sh b/src/main/resources/toGit/migration/sources/ccm/git-remove-all-git-related-files-2plus-levels.sh new file mode 100644 index 0000000..6a60205 --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/git-remove-all-git-related-files-2plus-levels.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -u +set -e +set -o pipefail +set -o posix + +[[ ${debug:-} == "true" ]] && set -x +cd $1 +pwd +echo "Files recursively named .gitignore .gitattributes .gitmodules" + +find . -mindepth 2 -type f -name '.gitignore' -o -name '.gitmodules' -o -name '.gitattributes' > remove_files.tmp || touch remove_files.tmp + +IFS=$'\n\r' +for file in $(cat remove_files.tmp) ; do + echo "remove file: $file" + [[ ${dryrun:-} == true ]] || rm -f "$file" +done + +rm -f remove_files.tmp diff --git a/src/main/resources/toGit/migration/sources/ccm/git-set-execute-bit-in-index-of-extensions.sh b/src/main/resources/toGit/migration/sources/ccm/git-set-execute-bit-in-index-of-extensions.sh new file mode 100644 index 0000000..e2e754f --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/git-set-execute-bit-in-index-of-extensions.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -u +set -e +set -o pipefail +set -o posix + +[[ ${debug:-} == "true" ]] && set -x + +script_dir=$(dirname $(readlink -f $0)) +source $(dirname $0)/_git-functions.sh || source ./_git-functions.sh + +cd $1 +git_set_execute_bit_in_index_of_extensions diff --git a/src/main/resources/toGit/migration/sources/ccm/git-set-execute-bit-in-index-of-unix-tool-file-executable.sh b/src/main/resources/toGit/migration/sources/ccm/git-set-execute-bit-in-index-of-unix-tool-file-executable.sh new file mode 100644 index 0000000..9ea2189 --- /dev/null +++ b/src/main/resources/toGit/migration/sources/ccm/git-set-execute-bit-in-index-of-unix-tool-file-executable.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -u +set -e +set -o pipefail +set -o posix + +[[ ${debug:-} == "true" ]] && set -x + +script_dir=$(dirname $(readlink -f $0)) +source $(dirname $0)/_git-functions.sh || source ./_git-functions.sh + +cd "$1" +exit_code=0 +git_set_execute_bit_in_index_of_unix_tool_file_executable || exit_code=$? +exit $exit_code