From 37c6478451beb9ba177ff2f0bc49af70493a3ebb Mon Sep 17 00:00:00 2001 From: Claus Schneider Date: Wed, 9 Apr 2025 13:34:57 +0200 Subject: [PATCH 1/6] Create git-retag.sh --- git-retag.sh | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 git-retag.sh diff --git a/git-retag.sh b/git-retag.sh new file mode 100644 index 0000000..1c8933a --- /dev/null +++ b/git-retag.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +# Enable strict mode for better error handling +set -euo pipefail + +while getopts ":o:n:dph" opt; do + case ${opt} in + o ) + tag_orig=$OPTARG + ;; + n ) + tag_new=$OPTARG + ;; + p ) + push_new_tag=true + ;; + d ) + delete_old_tag=true + ;; + \?|h ) + echo "Usage: $0 -o -n [-p [-d]]" + echo " -o Specify the old tag to be retagged" + echo " -n Specify the new tag name" + echo " -p Push the new tag to remote" + echo " -d Delete the old tag from remote after retagging: only if you use -p" + echo " -h Show this help message" + exit 1 + ;; + esac +done + +if [[ -z "${tag_orig:-}" ]] || [[ -z "${tag_new:-}" ]]; then + echo "Usage: $0 -o -n " + exit 1 +fi + +trap 'rm -f ./tag_meta_data.txt' EXIT + +function get_old_tag_info() { + + export GIT_AUTHOR_DATE="$(git tag -l --format="%(taggerdate:iso)" ${tag_orig})" + git tag -l --format="%(taggerdate:raw)" ${tag_orig} + export GIT_COMMITTER_DATE="${GIT_AUTHOR_DATE}" + + export GIT_AUTHOR_NAME=$(git tag -l --format="%(taggername)" ${tag_orig}) + export GIT_COMMITTER_NAME=${GIT_AUTHOR_NAME} + export GIT_AUTHOR_EMAIL=$(git tag -l --format="%(taggeremail)" ${tag_orig}) + export GIT_COMMITTER_EMAIL=${GIT_AUTHOR_EMAIL} + export GIT_TAGGER_NAME=${GIT_AUTHOR_NAME} + export GIT_TAGGER_EMAIL=${GIT_AUTHOR_EMAIL} + export GIT_TAGGER_DATE=${GIT_AUTHOR_DATE} + git tag -l --format '%(contents)' ${tag_orig} > ./tag_meta_data.txt +} + +function create_new_tag() { + + if [ ! -f ./tag_meta_data.txt ]; then + echo "Error: Metadata file not found. Cannot create new tag." + exit 1 + fi + + echo "Creating new tag ${tag_new} with data and message from ${tag_orig}" + git tag -f -a -F ./tag_meta_data.txt ${tag_new} ${tag_orig}^{} + git tag -l --format="%(taggerdate:iso)" ${tag_new} + git tag -l --format="%(taggerdate:raw)" ${tag_new} + +} + +function delete_old_tag_remotely() { + echo "Deleting old tag ${tag_orig} from remote" + git push origin :refs/tags/${tag_orig} + echo "Deleting old tag ${tag_orig} locally" + git tag -d ${tag_orig} +} + +function push_new_tag() { + echo "Pushing new tag ${tag_new} to remote" + git push origin refs/tags/${tag_new} +} + + +get_old_tag_info ${tag_orig} +create_new_tag ${tag_new} +push_new_tag +[[ ${push_new_tag:-false} == true ]] && push_new_tag ${tag_orig} && { + [[ ${delete_old_tag:-false} == true ]] && delete_old_tag_remotely ${tag_orig} +} From 8454dec99e201ace3abb3571444a607263598935 Mon Sep 17 00:00:00 2001 From: Claus Schneider Date: Tue, 22 Apr 2025 17:16:59 +0200 Subject: [PATCH 2/6] Update git retagger --- git-retag.sh => git-utils/git-retag.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename git-retag.sh => git-utils/git-retag.sh (95%) mode change 100644 => 100755 diff --git a/git-retag.sh b/git-utils/git-retag.sh old mode 100644 new mode 100755 similarity index 95% rename from git-retag.sh rename to git-utils/git-retag.sh index 1c8933a..54b29e9 --- a/git-retag.sh +++ b/git-utils/git-retag.sh @@ -30,7 +30,7 @@ while getopts ":o:n:dph" opt; do done if [[ -z "${tag_orig:-}" ]] || [[ -z "${tag_new:-}" ]]; then - echo "Usage: $0 -o -n " + $0 -h exit 1 fi @@ -38,6 +38,7 @@ trap 'rm -f ./tag_meta_data.txt' EXIT function get_old_tag_info() { + git show refs/tags/$tag_orig > /dev/null || git fetch --force origin --no-tags refs/tags/$tag_orig:refs/tags/$tag_orig export GIT_AUTHOR_DATE="$(git tag -l --format="%(taggerdate:iso)" ${tag_orig})" git tag -l --format="%(taggerdate:raw)" ${tag_orig} export GIT_COMMITTER_DATE="${GIT_AUTHOR_DATE}" @@ -81,7 +82,6 @@ function push_new_tag() { get_old_tag_info ${tag_orig} create_new_tag ${tag_new} -push_new_tag [[ ${push_new_tag:-false} == true ]] && push_new_tag ${tag_orig} && { [[ ${delete_old_tag:-false} == true ]] && delete_old_tag_remotely ${tag_orig} } From 8c3e2ec1e72e1dc7fb52563f9c52feaa4bd5fd5c Mon Sep 17 00:00:00 2001 From: Claus Schneider Date: Tue, 22 Apr 2025 17:53:45 +0200 Subject: [PATCH 3/6] Add git submodule usage in parent repo --- git-utils/git-retag.sh | 2 +- git-utils/git-submodule-usage.sh | 62 ++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100755 git-utils/git-submodule-usage.sh diff --git a/git-utils/git-retag.sh b/git-utils/git-retag.sh index 54b29e9..7b4eaa2 100755 --- a/git-utils/git-retag.sh +++ b/git-utils/git-retag.sh @@ -6,7 +6,7 @@ set -euo pipefail while getopts ":o:n:dph" opt; do case ${opt} in o ) - tag_orig=$OPTARG + tag_orig=$OPTARG ;; n ) tag_new=$OPTARG diff --git a/git-utils/git-submodule-usage.sh b/git-utils/git-submodule-usage.sh new file mode 100755 index 0000000..0553bdf --- /dev/null +++ b/git-utils/git-submodule-usage.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Enable strict mode for better error handling +set -euo pipefail + +while getopts "s:h" opt; do + case ${opt} in + s ) + subm_repo_name="${OPTARG}" + shift + ;; + \?|h ) + echo "Usage: $0 -s " + echo " -s Specify the old tag to be retagged" + exit 1 + ;; + esac +done + +if [[ -z "${subm_repo_name:-}" ]]; then + $0 -h + exit 1 +fi + +for subm_path in $(git config get -f .gitmodules --all --regexp ".*/${subm_repo_name}.path$"); do + printf "Submodule path: %s\n" "$subm_path" + for sha1_root in $(git rev-list --all -- ${subm_path}) ; do + sha1_sub="$(git ls-tree $sha1_root ${subm_path} | cut -d " " -f 3 | cut -d$'\t' -f 1 )" || true + + if [ -z "${sha1_sub:-}" ]; then + sha1_sub="(no submodule)" + else + git -C ${subm_path} rev-parse --verify $sha1_sub > /dev/null 2>&1 || { + git -C ${subm_path} fetch origin $sha1_sub > /dev/null + git artifact fetch-tags -s $sha1_sub > /dev/null + git -C ${subm_path} rev-parse --verify $sha1_sub > /dev/null 2>&1 || { + sha1_sub_ls_remote="(dead sha1)" + } + } + + sha1_sub_ls_remote=$(git -C ${subm_path} ls-remote --tags origin | grep $sha1_sub | cut -d / -f 3-) || true + + if [ -z "${sha1_sub_ls_remote:-}" ]; then + sha1_sub_ls_remote="(no tag)" + fi + fi + + remote_branches_contains_count=$(git branch -r --contains $sha1_root | wc -l) + tags_contains_count=$(git tag --contains $sha1_root | wc -l) + + printf "%14.14s %-60.60s %-80.80s %-20.20s\n" \ + "$sha1_sub" \ + "$sha1_sub_ls_remote" \ + "$(git log --oneline -1 --decorate --format="%h %cd %s" $sha1_root | cut -c1-90)" \ + "(ct.br:$remote_branches_contains_count ct.t:${tags_contains_count})" + if [[ "${verbose_branches:-verbose_branches_false}" == true && "$remote_branches_contains_count" -gt 0 ]]; then + git branch -r --contains $sha1_root | grep -e origin/master -e origin/main -e origin/products/.* || true + fi + + done + echo +done From 5f6bcf5e4836aa03666cc519e358285ff8e22f6b Mon Sep 17 00:00:00 2001 From: Claus Schneider Date: Fri, 16 May 2025 14:45:32 +0200 Subject: [PATCH 4/6] Create git-batch-push-process-pull.sh --- git-utils/git-batch-push-process-pull.sh | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 git-utils/git-batch-push-process-pull.sh diff --git a/git-utils/git-batch-push-process-pull.sh b/git-utils/git-batch-push-process-pull.sh new file mode 100644 index 0000000..9916d6d --- /dev/null +++ b/git-utils/git-batch-push-process-pull.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +echo "git checkout master" +git checkout master + +echo "git creating new local branch from timestamp" +INPUT_BRANCH_NAME=$(date +%s) +git checkout -b $INPUT_BRANCH_NAME + +echo "Add new files to the staging area" +git add . + +echo "Commit the new files" +git commit -m "Add new files" + +echo "New files committed:" +git diff --cached --name-only + +git push -u origin $INPUT_BRANCH_NAME + +echo "Branch '$INPUT_BRANCH_NAME' pushed to remote." + +echo "waiting for output from pipeline..." + +#it will have name like refs/heads/1739360601_20250212.24_output +echo -e "\n\n\tif pipeline take longer than 60s check this page\n\t\n\n" +while ! git ls-remote --exit-code --heads origin refs/heads/$INPUT_BRANCH_NAME*_output +do echo 'Hit CTRL+C to stop';sleep 5; done + +if git ls-remote --exit-code --heads origin refs/heads/$INPUT_BRANCH_NAME*_output 2>&1 1>/dev/null +then + OUTPUT_BRANCH_NAME=$(git ls-remote --exit-code --heads origin refs/heads/$INPUT_BRANCH_NAME*_output| cut -d/ -f3-) + echo "pipeline created $OUTPUT_BRANCH_NAME branch, running git pull and checkout" + git pull 2>&1 1>/dev/null + + git checkout $OUTPUT_BRANCH_NAME + git checkout master + #it has strange bug, after git pull and checkout to master, git does not see files, so im checkouting to output branch, then to master + + git checkout $OUTPUT_BRANCH_NAME audio_files/*.wav 2>&1 1>/dev/null + git checkout $OUTPUT_BRANCH_NAME audio_files/*.txt 2>&1 1>/dev/null + + echo "removing remote input branch" + git push origin --delete $INPUT_BRANCH_NAME + +else + echo "remote branch not found" + exit -1 +fi From 5f09f40b47a77e27a8df3f2502cdbf0edfd07c58 Mon Sep 17 00:00:00 2001 From: Claus Schneider Date: Wed, 3 Sep 2025 13:16:44 +0200 Subject: [PATCH 5/6] Add Jenkins backup XML script --- jenkins-tricks-examples/jenkins-backup-xml.sh | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 jenkins-tricks-examples/jenkins-backup-xml.sh diff --git a/jenkins-tricks-examples/jenkins-backup-xml.sh b/jenkins-tricks-examples/jenkins-backup-xml.sh new file mode 100644 index 0000000..c896d15 --- /dev/null +++ b/jenkins-tricks-examples/jenkins-backup-xml.sh @@ -0,0 +1,50 @@ +#!/bin/bash +set -xeu + +export WORKSPACE_CYGPATH=$(pwd) + +if [[ -d xml ]]; then + cd xml + git reset --hard + ls -1 | xargs rm -rf +else + mkdir xml + cd xml + git init + git remote add origin ${git_remote} + echo "**/configurations/" > .gitignore + echo "**/builds/" >> .gitignore + echo "**/fingerprints/" >> .gitignore + echo "**/*_deleted_*/" >> .gitignore +fi +cd $jenkins_path +find . -name "*.xml" -exec cp --parents \{\} ${WORKSPACE_CYGPATH}/xml \; +cd - + +cd ${WORKSPACE_CYGPATH}/xml + +git add -A . +git commit -m "$(date) - $jenkins_path ${tag}" || echo "never mind" +git push origin -f master:master-xml +if [[ ${tag:-} != "" ]]; then + git tag -a -m "${tag}" ${tag} -f + git push origin ${tag} -f +fi + +cd ${WORKSPACE_CYGPATH}/xml/jobs + +if [ ${make_disabled:-} == true ]; then + IFS=$'\n\r' + for config_file in $( find . -maxdepth 4 -name config.xml ); do + echo "${config_file}" + cat $config_file | \ + sed -e 's/ false<\/disabled>/ true<\/disabled>/' \ + > "${config_file}_tmp" + mv "${config_file}_tmp" "${config_file}" + grep -q "^ " "${config_file}" || echo "Likely folder" + done + git add -A . + git commit -m "$(date) - disabled" || echo "never mind" + git push origin -f master:master-disabled-xml +fi + From 2e2f199c063457048aec5d1a878c49df2e602d8a Mon Sep 17 00:00:00 2001 From: Claus Schneider Date: Tue, 16 Sep 2025 11:10:32 +0200 Subject: [PATCH 6/6] Enhance jenkins backup script with git fetch and reset and effect find --- jenkins-tricks-examples/jenkins-backup-xml.sh | 90 +++++++++++++++---- 1 file changed, 73 insertions(+), 17 deletions(-) diff --git a/jenkins-tricks-examples/jenkins-backup-xml.sh b/jenkins-tricks-examples/jenkins-backup-xml.sh index c896d15..cfee923 100644 --- a/jenkins-tricks-examples/jenkins-backup-xml.sh +++ b/jenkins-tricks-examples/jenkins-backup-xml.sh @@ -1,37 +1,93 @@ #!/bin/bash set -xeu -export WORKSPACE_CYGPATH=$(pwd) +export WORKSPACE=$(pwd) if [[ -d xml ]]; then cd xml - git reset --hard - ls -1 | xargs rm -rf + git fetch origin -p + git reset --hard origin/$(hostname)-master-xml || git reset --hard origin/master-xml || git reset --hard origin/master + ls -1 | xargs rm -rf else mkdir xml - cd xml - git init - git remote add origin ${git_remote} - echo "**/configurations/" > .gitignore - echo "**/builds/" >> .gitignore - echo "**/fingerprints/" >> .gitignore - echo "**/*_deleted_*/" >> .gitignore + cd xml + git init + git remote add origin ${REPO_URL} + git fetch origin -p + git reset --hard origin $(hostname)-master-xml fi + +# system and plugins +echo "**/configurations/" > .gitignore +echo "**/builds/" >> .gitignore +echo "**/fingerprints/" >> .gitignore +echo "**/*_deleted_*/" >> .gitignore +echo "plugins/*" >> .gitignore +echo "users/*" >> .gitignore +echo "nodes/*" >> .gitignore +echo "config-history/*" >> .gitignore +echo "scm-sync-configuration/*" >> .gitignore +echo "workspace/*" >> .gitignore + +# user path for clean up +echo "jobs/esw/jobs/bitbucket/*" >> .gitignore +echo "jobs/esw/jobs/sonarcloud/*" >> .gitignore +echo "config.bak/*" >> .gitignore + cd $jenkins_path -find . -name "*.xml" -exec cp --parents \{\} ${WORKSPACE_CYGPATH}/xml \; +find . \ + -path "*/configurations" -prune -o \ + -path "*/builds" -prune -o \ + -path "*/fingerprints" -prune -o \ + -path "*/*_deleted_*" -prune -o \ + -path "./plugins/*" -prune -o \ + -path "./users/*" -prune -o \ + -path "./nodes/*" -prune -o \ + -path "./config-history/*" -prune -o \ + -path "./scm-sync-configuration/*" -prune -o \ + -path "./workspace/*" -prune -o \ + \ + -path "./jobs/esw/jobs/bitbucket/*" -prune -o \ + -path "./jobs/esw/jobs/sonarcloud/*" -prune -o \ + -name 'config.xml' -type f \ + -print + +[[ ${dryrun:-} == true ]] && exit + +find . \ + -path "*/configurations" -prune -o \ + -path "*/builds" -prune -o \ + -path "*/fingerprints" -prune -o \ + -path "*/*_deleted_*" -prune -o \ + -path "./plugins/*" -prune -o \ + -path "./users/*" -prune -o \ + -path "./nodes/*" -prune -o \ + -path "./config-history/*" -prune -o \ + -path "./scm-sync-configuration/*" -prune -o \ + -path "./workspace/*" -prune -o \ + \ + -path "./jobs/esw/jobs/bitbucket/*" -prune -o \ + -path "./jobs/esw/jobs/sonarcloud/*" -prune -o \ + -name 'config.xml' -type f \ + -exec cp --parents \{\} ${WORKSPACE}/xml \; cd - -cd ${WORKSPACE_CYGPATH}/xml +cd ${WORKSPACE}/xml git add -A . -git commit -m "$(date) - $jenkins_path ${tag}" || echo "never mind" -git push origin -f master:master-xml +if [[ ${tag:-} == "" ]]; then + tag=$(date +"%Y-%m-%dT%H-%M-%S") +fi +tag_commit="$(hostname)-${jenkins_path}-${tag}" +git commit -m "${tag_commit}" +git push origin -f HEAD:$(hostname)-master-xml if [[ ${tag:-} != "" ]]; then - git tag -a -m "${tag}" ${tag} -f - git push origin ${tag} -f + git tag -a -m "${tag_commit}" ${tag_commit} -f + git push origin ${tag_commit} -f fi -cd ${WORKSPACE_CYGPATH}/xml/jobs + +cd ${WORKSPACE}/xml/jobs if [ ${make_disabled:-} == true ]; then IFS=$'\n\r'