From c98724b0228b892b738f503434bdc28463ba488c Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 9 Aug 2019 00:14:11 +1000 Subject: [PATCH 1/9] scripts/git: Cleanup and fix minor git scripts issues Fix typos, clean up formatting, rewrite some comments, add headings. Preparation for 31314. --- scripts/git/git-merge-forward.sh | 75 +++++++++++++++++++------------- scripts/git/git-push-all.sh | 18 +++++++- 2 files changed, 61 insertions(+), 32 deletions(-) diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index ba29983284..e2b5bde49c 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -1,9 +1,11 @@ #!/usr/bin/env bash -############################## -# Configuration (change me!) # -############################## +################# +# Configuration # +################# +# Don't change this configuration - set the env vars in your .profile +# # The general setup that is suggested here is: # # GIT_PATH = /home//git/ @@ -21,20 +23,23 @@ TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} -######################### -# End of configuration. # -######################### +########################## +# Git branches to manage # +########################## + +# The branches and worktrees need to be modified when there is a new branch, +# and when an old branch is no longer supported. # Configuration of the branches that needs merging. The values are in order: -# (1) Branch name that we merge onto. -# (2) Branch name to merge from. In other words, this is merge into (1) -# (3) Full path of the git worktree. +# (0) current maint/release branch name +# (1) previous maint/release name to merge into (0) +# (2) Full path of the git worktree # # As an example: -# $ cd (3) -# $ git checkout maint-0.3.5 (1) +# $ cd (2) +# $ git checkout maint-0.3.5 (0) # $ git pull -# $ git merge maint-0.3.4 (2) +# $ git merge maint-0.3.4 (1) # # First set of arrays are the maint-* branch and then the release-* branch. # New arrays need to be in the WORKTREE= array else they aren't considered. @@ -65,9 +70,28 @@ ${RELEASE_040[0]} ${RELEASE_041[0]} EOF -########################## -# Git Worktree to manage # -########################## +####################### +# Argument processing # +####################### + +# Controlled by the -n option. The dry run option will just output the command +# that would have been executed for each worktree. +DRY_RUN=0 + +while getopts "n" opt; do + case "$opt" in + n) DRY_RUN=1 + echo " *** DRY RUN MODE ***" + ;; + *) + exit 1 + ;; + esac +done + +########################### +# Git worktrees to manage # +########################### # List of all worktrees to work on. All defined above. Ordering is important. # Always the maint-* branch BEFORE then the release-*. @@ -87,9 +111,9 @@ WORKTREE=( ) COUNT=${#WORKTREE[@]} -# Controlled by the -n option. The dry run option will just output the command -# that would have been executed for each worktree. -DRY_RUN=0 +############# +# Constants # +############# # Control characters CNRM=$'\x1b[0;0m' # Clear color @@ -150,7 +174,7 @@ function pull_branch fi } -# Merge the given branch name ($2) into the current branch ($1). +# Merge the given branch name ($1) into the current branch ($2). function merge_branch { local cmd="git merge --no-edit $1" @@ -203,16 +227,6 @@ function fetch_origin # Entry point # ############### -while getopts "n" opt; do - case "$opt" in - n) DRY_RUN=1 - echo " *** DRY DRUN MODE ***" - ;; - *) - ;; - esac -done - # First, fetch the origin. goto_repo "$ORIGIN_PATH" fetch_origin @@ -231,6 +245,7 @@ for ((i=0; i # env vars: TOR_UPSTREAM_REMOTE_NAME=upstream TOR_PUSH_DELAY=0 -# options: --no-atomic --dry-run (any other git push option) +# git-opts: --no-atomic --dry-run (any other git push option) # # TOR_PUSH_DELAY pushes the master and maint branches separately, so that CI # runs in a sensible order. @@ -10,11 +10,21 @@ set -e +################# +# Configuration # +################# + +# Don't change this configuration - set the env vars in your .profile +# # The upstream remote which git.torproject.org/tor.git points to. UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} # Add a delay between pushes, so CI runs on the most important branches first PUSH_DELAY=${TOR_PUSH_DELAY:-0} +######################## +# Git branches to push # +######################## + PUSH_BRANCHES=$(echo \ master \ {release,maint}-0.4.1 \ @@ -23,6 +33,10 @@ PUSH_BRANCHES=$(echo \ {release,maint}-0.2.9 \ ) +############### +# Entry point # +############### + if [ "$PUSH_DELAY" -le 0 ]; then echo "Pushing $PUSH_BRANCHES" # We know that there are no spaces in any branch within $PUSH_BRANCHES, so From 667311ebbd6fe6c43da8eb5abc1e34e30fb5b911 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 9 Aug 2019 00:18:56 +1000 Subject: [PATCH 2/9] scripts/git: Add test branch support to the git scripts Add a -t argument to git-merge-forward.sh and git-push-all.sh, which makes these scripts create, merge forward, and push test branches. Add a -r argument to git-push-all.sh, so the script can push test branches to a personal remote. Closes ticket 31314. --- changes/ticket31314 | 6 ++ scripts/git/git-merge-forward.sh | 150 ++++++++++++++++++++++++++----- scripts/git/git-push-all.sh | 94 +++++++++++++++++-- 3 files changed, 218 insertions(+), 32 deletions(-) create mode 100644 changes/ticket31314 diff --git a/changes/ticket31314 b/changes/ticket31314 new file mode 100644 index 0000000000..e1d9fb767b --- /dev/null +++ b/changes/ticket31314 @@ -0,0 +1,6 @@ + o Minor features (git scripts): + - Add a -t argument to git-merge-forward.sh and + git-push-all.sh, which makes these scripts create, merge forward, and + push test branches. Closes ticket 31314. + - Add a -r argument to git-push-all.sh, so the script can + push test branches to a personal remote. Closes ticket 31314. diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index e2b5bde49c..1f4ddb85e7 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -33,20 +33,45 @@ TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} # Configuration of the branches that needs merging. The values are in order: # (0) current maint/release branch name # (1) previous maint/release name to merge into (0) +# (only used in merge forward mode) # (2) Full path of the git worktree +# (3) current branch suffix +# (maint branches only, only used in test branch mode) +# (4) previous test branch suffix to merge into (3) +# (maint branches only, only used in test branch mode) # -# As an example: +# Merge forward example: # $ cd (2) # $ git checkout maint-0.3.5 (0) # $ git pull # $ git merge maint-0.3.4 (1) # +# Test branch example: +# $ cd (2) +# $ git checkout -b ticket99999_035 (3) +# $ git checkout maint-0.3.5 (0) +# $ git pull +# $ git checkout ticket99999_035 +# $ git merge maint-0.3.5 +# $ git merge ticket99999_034 (4) +# # First set of arrays are the maint-* branch and then the release-* branch. # New arrays need to be in the WORKTREE= array else they aren't considered. -MAINT_035=( "maint-0.3.5" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) -MAINT_040=( "maint-0.4.0" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) -MAINT_041=( "maint-0.4.1" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" ) -MAINT_MASTER=( "master" "maint-0.4.1" "$GIT_PATH/$TOR_MASTER_NAME" ) +# +# Only used in test branch mode +# There is no previous branch to merge forward, so the second and fifth items +# must be blank ("") +MAINT_029_TB=( "maint-0.2.9" "" "$GIT_PATH/$TOR_WKT_NAME/maint-0.2.9" \ + "_029" "") +# Used in maint/release merge and test branch modes +MAINT_035=( "maint-0.3.5" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" \ + "_035" "_029") +MAINT_040=( "maint-0.4.0" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" \ + "_040" "_035") +MAINT_041=( "maint-0.4.1" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" \ + "_041" "_040") +MAINT_MASTER=( "master" "maint-0.4.1" "$GIT_PATH/$TOR_MASTER_NAME" \ + "_master" "_041") RELEASE_029=( "release-0.2.9" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) RELEASE_035=( "release-0.3.5" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) @@ -60,6 +85,7 @@ ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" # SC2034 -- shellcheck thinks that these are unused. We know better. ACTUALLY_THESE_ARE_USED=< option. The test branch base +# name option makes git-merge-forward.sh create new test branches: +# _029, _035, ... , _master, and merge forward. +TEST_BRANCH_PREFIX= + +while getopts "nt:" opt; do case "$opt" in n) DRY_RUN=1 echo " *** DRY RUN MODE ***" ;; + t) TEST_BRANCH_PREFIX="$OPTARG" + echo " *** CREATING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***" + ;; *) exit 1 ;; @@ -93,22 +127,44 @@ done # Git worktrees to manage # ########################### -# List of all worktrees to work on. All defined above. Ordering is important. -# Always the maint-* branch BEFORE then the release-*. -WORKTREE=( - RELEASE_029[@] +if [ -z "$TEST_BRANCH_PREFIX" ]; then - MAINT_035[@] - RELEASE_035[@] + # maint/release merge mode + # + # List of all worktrees to work on. All defined above. Ordering is important. + # Always the maint-* branch BEFORE then the release-*. + WORKTREE=( + RELEASE_029[@] - MAINT_040[@] - RELEASE_040[@] + MAINT_035[@] + RELEASE_035[@] - MAINT_041[@] - RELEASE_041[@] + MAINT_040[@] + RELEASE_040[@] + + MAINT_041[@] + RELEASE_041[@] + + MAINT_MASTER[@] + ) + +else + + # Test branch mode: merge to maint only, and create a new branch for 0.2.9 + WORKTREE=( + MAINT_029_TB[@] + + MAINT_035[@] + + MAINT_040[@] + + MAINT_041[@] + + MAINT_MASTER[@] + ) + +fi - MAINT_MASTER[@] -) COUNT=${#WORKTREE[@]} ############# @@ -161,6 +217,19 @@ function switch_branch fi } +# Checkout a new branch with the given branch name. +function new_branch +{ + local cmd="git checkout -b $1" + printf " %s Creating new branch %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + # Pull the given branch name. function pull_branch { @@ -236,16 +305,51 @@ for ((i=0; i +# Usage: git-push-all.sh -t -r # env vars: TOR_UPSTREAM_REMOTE_NAME=upstream TOR_PUSH_DELAY=0 # git-opts: --no-atomic --dry-run (any other git push option) # @@ -17,10 +17,49 @@ set -e # Don't change this configuration - set the env vars in your .profile # # The upstream remote which git.torproject.org/tor.git points to. +# In test branch mode, override this setting with -r UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} # Add a delay between pushes, so CI runs on the most important branches first PUSH_DELAY=${TOR_PUSH_DELAY:-0} +####################### +# Argument processing # +####################### + +# Controlled by the -t option. The test branch base +# name option makes git-merge-forward.sh create new test branches: +# _029, _035, ... , _master, and merge forward. +TEST_BRANCH_PREFIX= + +while getopts ":r:t:" opt; do + case "$opt" in + r) UPSTREAM_REMOTE="$OPTARG" + echo " *** PUSHING TO REMOTE: ${UPSTREAM_REMOTE} ***" + shift + shift + OPTIND=$[$OPTIND - 2] + ;; + t) TEST_BRANCH_PREFIX="$OPTARG" + echo " *** PUSHING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***" + shift + shift + OPTIND=$[$OPTIND - 2] + ;; + *) + # Assume git push will handle the option + ;; + esac +done + +if [ "$TEST_BRANCH_PREFIX" ]; then + if [ "$UPSTREAM_REMOTE" = ${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} ]; then + echo "Pushing test branches ${TEST_BRANCH_PREFIX}_nnn to " \ + "$UPSTREAM_REMOTE is not allowed." + echo "Usage: $0 -r -t " + exit 1 + fi +fi + ######################## # Git branches to push # ######################## @@ -33,6 +72,32 @@ PUSH_BRANCHES=$(echo \ {release,maint}-0.2.9 \ ) +if [ -z "$TEST_BRANCH_PREFIX" ]; then + + # maint/release push mode + # + # List of branches to push. Ordering is not important. + PUSH_BRANCHES=$(echo \ + master \ + {release,maint}-0.4.1 \ + {release,maint}-0.4.0 \ + {release,maint}-0.3.5 \ + {release,maint}-0.2.9 \ + ) +else + + # Test branch mode: merge to maint only, and create a new branch for 0.2.9 + # + # List of branches to push. Ordering is not important. + PUSH_BRANCHES=$(echo \ + ${TEST_BRANCH_PREFIX}_master \ + ${TEST_BRANCH_PREFIX}_041 \ + ${TEST_BRANCH_PREFIX}_040 \ + ${TEST_BRANCH_PREFIX}_035 \ + ${TEST_BRANCH_PREFIX}_029 \ + ) +fi + ############### # Entry point # ############### @@ -48,18 +113,29 @@ if [ "$PUSH_DELAY" -le 0 ]; then else PUSH_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | sort -V) MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep master) - MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep maint) - RELEASE_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep release | \ - tr "\n" " ") - printf "Pushing with %ss delays, so CI runs in this order:\n%s\n%s\n%s\n" \ - "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES" "$RELEASE_BRANCHES" + if [ -z "$TEST_BRANCH_PREFIX" ]; then + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep maint) + RELEASE_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep release | \ + tr "\n" " ") + printf "Pushing with %ss delays, so CI runs in this order:\n%s\n%s\n%s\n" \ + "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES" "$RELEASE_BRANCHES" + else + # Actually test branches based on maint branches + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep -v master) + printf "Pushing with %ss delays, so CI runs in this order:\n%s\n%s\n" \ + "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES" + # No release branches + RELEASE_BRANCHES= + fi git push "$@" "$UPSTREAM_REMOTE" "$MASTER_BRANCH" sleep "$PUSH_DELAY" # shellcheck disable=SC2086 for b in $MAINT_BRANCHES; do - git push "$@" "$UPSTREAM_REMOTE" $b + git push "$@" "$UPSTREAM_REMOTE" "$b" sleep "$PUSH_DELAY" done - # shellcheck disable=SC2086 - git push --atomic "$@" "$UPSTREAM_REMOTE" $RELEASE_BRANCHES + if [ "$RELEASE_BRANCHES" ]; then + # shellcheck disable=SC2086 + git push --atomic "$@" "$UPSTREAM_REMOTE" $RELEASE_BRANCHES + fi fi From 15782758c7c9e170fccc138a0c4897a602217641 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 9 Aug 2019 14:37:38 +1000 Subject: [PATCH 3/9] scripts/git: Allow git-merge-forward.sh to re-use existing test branches Add a -u argument to git-merge-forward.sh, so that the script can re-use existing test branches after a merge failure and fix. Part of 31314. --- changes/ticket31314 | 3 +++ scripts/git/git-merge-forward.sh | 46 +++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/changes/ticket31314 b/changes/ticket31314 index e1d9fb767b..2932231171 100644 --- a/changes/ticket31314 +++ b/changes/ticket31314 @@ -4,3 +4,6 @@ push test branches. Closes ticket 31314. - Add a -r argument to git-push-all.sh, so the script can push test branches to a personal remote. Closes ticket 31314. + - Add a -u argument to git-merge-forward.sh, so that the script can re-use + existing test branches after a merge failure and fix. + Closes ticket 31314. diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index 1f4ddb85e7..a7e797eaf0 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -109,7 +109,12 @@ DRY_RUN=0 # _029, _035, ... , _master, and merge forward. TEST_BRANCH_PREFIX= -while getopts "nt:" opt; do +# Controlled by the -u option. The use existing option checks for existing +# branches with the , and checks them out, rather than +# creating a new branch. +USE_EXISTING=0 + +while getopts "nt:u" opt; do case "$opt" in n) DRY_RUN=1 echo " *** DRY RUN MODE ***" @@ -117,6 +122,9 @@ while getopts "nt:" opt; do t) TEST_BRANCH_PREFIX="$OPTARG" echo " *** CREATING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***" ;; + u) USE_EXISTING=1 + echo " *** USE EXISTING TEST BRANCHES MODE ***" + ;; *) exit 1 ;; @@ -230,6 +238,32 @@ function new_branch fi } +# Switch to an existing branch, or checkout a new branch with the given +# branch name. +function switch_or_new_branch +{ + local cmd="git rev-parse --verify $1" + if [ $DRY_RUN -eq 0 ]; then + # Call switch_branch if there is a branch, or new_branch if there is not + msg=$( eval "$cmd" 2>&1 ) + RET=$? + if [ $RET -eq 0 ]; then + # Branch: (commit id) + switch_branch "$1" + elif [ $RET -eq 128 ]; then + # Not a branch: "fatal: Needed a single revision" + new_branch "$1" + else + # Unexpected return value + validate_ret $RET "$msg" + fi + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}, then depending on the result:" + switch_branch "$1" + new_branch "$1" + fi +} + # Pull the given branch name. function pull_branch { @@ -328,8 +362,14 @@ for ((i=0; i Date: Mon, 12 Aug 2019 11:10:12 +1000 Subject: [PATCH 4/9] scripts/git: Make the git push command and args configurable TOR_GIT_PUSH provides the git push command and default arguments. Also fix handling of git-push-all.sh script arguments and arguments that are passed through to $TOR_GIT_PUSH, using a "--" argument as a separator. Fix on 29879. --- changes/ticket31314 | 5 +++++ scripts/git/git-push-all.sh | 27 +++++++++++++++++++++------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/changes/ticket31314 b/changes/ticket31314 index 2932231171..8b5f5da8f9 100644 --- a/changes/ticket31314 +++ b/changes/ticket31314 @@ -7,3 +7,8 @@ - Add a -u argument to git-merge-forward.sh, so that the script can re-use existing test branches after a merge failure and fix. Closes ticket 31314. + - Add a TOR_GIT_PUSH env var, which sets the default git push command and + arguments for git-push-all.sh. Closes ticket 31314. + - Add a "--" command-line argument, to + separate git-push-all.sh script arguments from arguments that are passed + through to git push. Closes ticket 31314. diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index f3bbe8b778..2d6e77a8c1 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# Usage: git-push-all.sh -t -r +# Usage: git-push-all.sh -t -r +# -- # env vars: TOR_UPSTREAM_REMOTE_NAME=upstream TOR_PUSH_DELAY=0 # git-opts: --no-atomic --dry-run (any other git push option) # @@ -16,6 +17,8 @@ set -e # Don't change this configuration - set the env vars in your .profile # +# git push command and default arguments +GIT_PUSH=${TOR_GIT_PUSH:-"git push --atomic"} # The upstream remote which git.torproject.org/tor.git points to. # In test branch mode, override this setting with -r UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} @@ -46,11 +49,21 @@ while getopts ":r:t:" opt; do OPTIND=$[$OPTIND - 2] ;; *) - # Assume git push will handle the option + # Assume we're done with script arguments, + # and git push will handle the option + break ;; esac done +# getopts doesn't allow "-" as an option character, +# so we have to handle -- manually +if [ "$1" = "--" ]; then + shift +fi + +echo "Calling git push --atomic $@ " + if [ "$TEST_BRANCH_PREFIX" ]; then if [ "$UPSTREAM_REMOTE" = ${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} ]; then echo "Pushing test branches ${TEST_BRANCH_PREFIX}_nnn to " \ @@ -108,9 +121,11 @@ if [ "$PUSH_DELAY" -le 0 ]; then # it is safe to use it unquoted. (This also applies to the other shellcheck # exceptions below.) # + # Push all the branches at the same time # shellcheck disable=SC2086 - git push --atomic "$@" "$UPSTREAM_REMOTE" $PUSH_BRANCHES + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $PUSH_BRANCHES else + # Push the branches in optimal CI order, with a delay between each push PUSH_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | sort -V) MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep master) if [ -z "$TEST_BRANCH_PREFIX" ]; then @@ -127,15 +142,15 @@ else # No release branches RELEASE_BRANCHES= fi - git push "$@" "$UPSTREAM_REMOTE" "$MASTER_BRANCH" + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$MASTER_BRANCH" sleep "$PUSH_DELAY" # shellcheck disable=SC2086 for b in $MAINT_BRANCHES; do - git push "$@" "$UPSTREAM_REMOTE" "$b" + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$b" sleep "$PUSH_DELAY" done if [ "$RELEASE_BRANCHES" ]; then # shellcheck disable=SC2086 - git push --atomic "$@" "$UPSTREAM_REMOTE" $RELEASE_BRANCHES + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $RELEASE_BRANCHES fi fi From b47b71ad2fc095b43437317011d127424188eb4f Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 12 Aug 2019 11:12:41 +1000 Subject: [PATCH 5/9] scripts/git: Let git-push-all.sh skip unchanged test branches Skip test branches that are the same as remote maint/release/master branches. Add a TOR_PUSH_SAME and -s argument to git-push-all.sh to change this default. Part of 31314. --- changes/ticket31314 | 4 ++ scripts/git/git-push-all.sh | 73 +++++++++++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/changes/ticket31314 b/changes/ticket31314 index 8b5f5da8f9..7ce96e96cf 100644 --- a/changes/ticket31314 +++ b/changes/ticket31314 @@ -12,3 +12,7 @@ - Add a "--" command-line argument, to separate git-push-all.sh script arguments from arguments that are passed through to git push. Closes ticket 31314. + - Skip pushing test branches that are the same as a remote + maint/release/master branch in git-push-all.sh by default. Add a -s + argument, so git-push-all.sh can push all test branches. + Closes ticket 31314. diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index 2d6e77a8c1..8a8ef1f2db 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Usage: git-push-all.sh -t -r +# Usage: git-push-all.sh -t -r -s # -- # env vars: TOR_UPSTREAM_REMOTE_NAME=upstream TOR_PUSH_DELAY=0 # git-opts: --no-atomic --dry-run (any other git push option) @@ -20,10 +20,17 @@ set -e # git push command and default arguments GIT_PUSH=${TOR_GIT_PUSH:-"git push --atomic"} # The upstream remote which git.torproject.org/tor.git points to. -# In test branch mode, override this setting with -r -UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} +DEFAULT_UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} +# Push to a different upstream remote using -r +UPSTREAM_REMOTE=${DEFAULT_UPSTREAM_REMOTE} # Add a delay between pushes, so CI runs on the most important branches first PUSH_DELAY=${TOR_PUSH_DELAY:-0} +# Push (1) or skip (0) test branches that are the same as an upstream +# maint/master branch. Push if you are testing that the CI environment still +# works on old code, skip if you are testing new code in the branch. +# Default: skip unchanged branches. +# Inverted by the -s option. +PUSH_SAME=${TOR_PUSH_SAME:-0} ####################### # Argument processing # @@ -34,7 +41,7 @@ PUSH_DELAY=${TOR_PUSH_DELAY:-0} # _029, _035, ... , _master, and merge forward. TEST_BRANCH_PREFIX= -while getopts ":r:t:" opt; do +while getopts ":r:st:" opt; do case "$opt" in r) UPSTREAM_REMOTE="$OPTARG" echo " *** PUSHING TO REMOTE: ${UPSTREAM_REMOTE} ***" @@ -42,6 +49,15 @@ while getopts ":r:t:" opt; do shift OPTIND=$[$OPTIND - 2] ;; + s) PUSH_SAME=$[! "$PUSH_SAME" ] + if [ "$PUSH_SAME" -eq 0 ]; then + echo " *** SKIPPING UNCHANGED TEST BRANCHES ***" + else + echo " *** PUSHING UNCHANGED TEST BRANCHES ***" + fi + shift + OPTIND=$[$OPTIND - 1] + ;; t) TEST_BRANCH_PREFIX="$OPTARG" echo " *** PUSHING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***" shift @@ -73,6 +89,29 @@ if [ "$TEST_BRANCH_PREFIX" ]; then fi fi +################################ +# Git upstream remote branches # +################################ + +DEFAULT_UPSTREAM_BRANCHES= +if [ "$DEFAULT_UPSTREAM_REMOTE" != "$UPSTREAM_REMOTE" ]; then + DEFAULT_UPSTREAM_BRANCHES=`echo \ + ${DEFAULT_UPSTREAM_REMOTE}/master \ + ${DEFAULT_UPSTREAM_REMOTE}/{release,maint}-0.4.1 \ + ${DEFAULT_UPSTREAM_REMOTE}/{release,maint}-0.4.0 \ + ${DEFAULT_UPSTREAM_REMOTE}/{release,maint}-0.3.5 \ + ${DEFAULT_UPSTREAM_REMOTE}/{release,maint}-0.2.9 \ + ` +fi + +UPSTREAM_BRANCHES=`echo \ + ${UPSTREAM_REMOTE}/master \ + ${UPSTREAM_REMOTE}/{release,maint}-0.4.1 \ + ${UPSTREAM_REMOTE}/{release,maint}-0.4.0 \ + ${UPSTREAM_REMOTE}/{release,maint}-0.3.5 \ + ${UPSTREAM_REMOTE}/{release,maint}-0.2.9 \ + ` + ######################## # Git branches to push # ######################## @@ -115,6 +154,32 @@ fi # Entry point # ############### +# Skip the test branches that are the same as the upstream branches +if [ "$PUSH_SAME" -eq 0 -a "$TEST_BRANCH_PREFIX" ]; then + NEW_PUSH_BRANCHES= + for b in $PUSH_BRANCHES; do + PUSH_COMMIT=`git rev-parse $b` + SKIP_UPSTREAM= + for u in $DEFAULT_UPSTREAM_BRANCHES $UPSTREAM_BRANCHES; do + UPSTREAM_COMMIT=`git rev-parse "$u"` + if [ "$PUSH_COMMIT" = "$UPSTREAM_COMMIT" ]; then + SKIP_UPSTREAM="$u" + fi + done + if [ "$SKIP_UPSTREAM" ]; then + printf "Skipping unchanged: %s remote: %s\n" \ + "$b" "$SKIP_UPSTREAM" + else + if [ "$NEW_PUSH_BRANCHES" ]; then + NEW_PUSH_BRANCHES="${NEW_PUSH_BRANCHES} ${b}" + else + NEW_PUSH_BRANCHES="${b}" + fi + fi + done + PUSH_BRANCHES=${NEW_PUSH_BRANCHES} +fi + if [ "$PUSH_DELAY" -le 0 ]; then echo "Pushing $PUSH_BRANCHES" # We know that there are no spaces in any branch within $PUSH_BRANCHES, so From 664e6a392ed0ace86e0182dc25bf365e8e34ed9d Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 29 Aug 2019 22:52:21 +1000 Subject: [PATCH 6/9] scripts/git: Improve usage documentation for merge-forward and push-all Part of 31314. --- scripts/git/git-merge-forward.sh | 12 ++++++++++++ scripts/git/git-push-all.sh | 25 +++++++++++++++++++------ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index a7e797eaf0..b9591eaa7e 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -1,5 +1,17 @@ #!/usr/bin/env bash +# Usage: git-merge-forward.sh -n -t -u +# arguments: +# -n: dry run mode +# -t: test branch mode: create new branches from the commits checked +# out in each maint directory. Call these branches prefix_029, +# prefix_035, ... , prefix_master. +# -u: in test branch mode, if a prefix_* branch exists, skip creating +# that branch. Use after a merge error, to restart the merge +# forward at the first unmerged branch. +# env vars: +# See the Configuration section for env vars and their default values. + ################# # Configuration # ################# diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index 8a8ef1f2db..0b3cfcd88e 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -2,12 +2,25 @@ # Usage: git-push-all.sh -t -r -s # -- -# env vars: TOR_UPSTREAM_REMOTE_NAME=upstream TOR_PUSH_DELAY=0 -# git-opts: --no-atomic --dry-run (any other git push option) -# -# TOR_PUSH_DELAY pushes the master and maint branches separately, so that CI -# runs in a sensible order. -# push --atomic is the default when TOR_PUSH_DELAY=0, and for release branches. +# arguments: +# -t: test branch mode: Push test branches, rather than maint and +# release branches. Pushes the branches called prefix_029, +# prefix_035, ... , prefix_master. +# -r: push to remote-name, rather than $TOR_UPSTREAM_REMOTE_NAME. +# -s: push branches whose tips match upstream maint, release, or +# master branches. The default is to skip these branches. Use +# -s when testing for CI environment failures with old code. +# --: pass any other arguments to git, rather than the script. +# env vars: +# TOR_GIT_PUSH: the git push command and arguments +# TOR_UPSTREAM_REMOTE_NAME: the default upstream, overridden by -r +# TOR_PUSH_DELAY: pushes the master and maint branches separately, +# so that CI runs in a sensible order. +# TOR_PUSH_SAME: push branches whose tips match upstream maint, +# release, or master branches. Inverted by -s. +# See the Configuration section for env var default values. +# git-opts: +# --no-atomic --dry-run (and any other git push option) set -e From d0e31b4d1f10279280ce7cc9ece71d17a79ed6b7 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 29 Aug 2019 22:53:44 +1000 Subject: [PATCH 7/9] scripts/git: Quote shell arguments where possible Most shell arguments should be quoted to avoid mistakes. But since all branch names are hard-coded, or supplied by the script user, we don't need to be too concerned about command injection. Quoting all shell arguments would take a major refactor. (Probably using arrays.) Part of 31314. --- scripts/git/git-merge-forward.sh | 10 +++++----- scripts/git/git-push-all.sh | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index b9591eaa7e..720320c9fe 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -227,7 +227,7 @@ function validate_ret # Switch to the given branch name. function switch_branch { - local cmd="git checkout $1" + local cmd="git checkout '$1'" printf " %s Switching branch to %s..." "$MARKER" "$1" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) @@ -240,7 +240,7 @@ function switch_branch # Checkout a new branch with the given branch name. function new_branch { - local cmd="git checkout -b $1" + local cmd="git checkout -b '$1'" printf " %s Creating new branch %s..." "$MARKER" "$1" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) @@ -254,7 +254,7 @@ function new_branch # branch name. function switch_or_new_branch { - local cmd="git rev-parse --verify $1" + local cmd="git rev-parse --verify '$1'" if [ $DRY_RUN -eq 0 ]; then # Call switch_branch if there is a branch, or new_branch if there is not msg=$( eval "$cmd" 2>&1 ) @@ -292,7 +292,7 @@ function pull_branch # Merge the given branch name ($1) into the current branch ($2). function merge_branch { - local cmd="git merge --no-edit $1" + local cmd="git merge --no-edit '$1'" printf " %s Merging branch %s into %s..." "$MARKER" "$1" "$2" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) @@ -305,7 +305,7 @@ function merge_branch # Pull the given branch name. function merge_branch_origin { - local cmd="git merge --ff-only origin/$1" + local cmd="git merge --ff-only 'origin/$1'" printf " %s Merging branch origin/%s..." "$MARKER" "$1" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index 0b3cfcd88e..37a73ec95f 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -171,7 +171,7 @@ fi if [ "$PUSH_SAME" -eq 0 -a "$TEST_BRANCH_PREFIX" ]; then NEW_PUSH_BRANCHES= for b in $PUSH_BRANCHES; do - PUSH_COMMIT=`git rev-parse $b` + PUSH_COMMIT=`git rev-parse "$b"` SKIP_UPSTREAM= for u in $DEFAULT_UPSTREAM_BRANCHES $UPSTREAM_BRANCHES; do UPSTREAM_COMMIT=`git rev-parse "$u"` From 340ff7f5f8c4e59cb55ea507ece273794be786e4 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 29 Aug 2019 22:56:19 +1000 Subject: [PATCH 8/9] scripts/git: fix an env var typo in git-merge-forward.sh Part of 31314. --- scripts/git/git-merge-forward.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index 720320c9fe..cbd2f3c3bd 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -369,7 +369,7 @@ for ((i=0; i Date: Thu, 29 Aug 2019 23:05:56 +1000 Subject: [PATCH 9/9] scripts/git: fix shellcheck issues in git-push-all.sh Part of 31314. --- scripts/git/git-push-all.sh | 50 ++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index 37a73ec95f..8e49e81b9d 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -60,22 +60,22 @@ while getopts ":r:st:" opt; do echo " *** PUSHING TO REMOTE: ${UPSTREAM_REMOTE} ***" shift shift - OPTIND=$[$OPTIND - 2] + OPTIND=$((OPTIND - 2)) ;; - s) PUSH_SAME=$[! "$PUSH_SAME" ] + s) PUSH_SAME=$((! PUSH_SAME)) if [ "$PUSH_SAME" -eq 0 ]; then echo " *** SKIPPING UNCHANGED TEST BRANCHES ***" else echo " *** PUSHING UNCHANGED TEST BRANCHES ***" fi shift - OPTIND=$[$OPTIND - 1] + OPTIND=$((OPTIND - 1)) ;; t) TEST_BRANCH_PREFIX="$OPTARG" echo " *** PUSHING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***" shift shift - OPTIND=$[$OPTIND - 2] + OPTIND=$((OPTIND - 2)) ;; *) # Assume we're done with script arguments, @@ -91,10 +91,10 @@ if [ "$1" = "--" ]; then shift fi -echo "Calling git push --atomic $@ " +echo "Calling $GIT_PUSH" "$@" "" if [ "$TEST_BRANCH_PREFIX" ]; then - if [ "$UPSTREAM_REMOTE" = ${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} ]; then + if [ "$UPSTREAM_REMOTE" = "${TOR_UPSTREAM_REMOTE_NAME:-upstream}" ]; then echo "Pushing test branches ${TEST_BRANCH_PREFIX}_nnn to " \ "$UPSTREAM_REMOTE is not allowed." echo "Usage: $0 -r -t " @@ -108,22 +108,22 @@ fi DEFAULT_UPSTREAM_BRANCHES= if [ "$DEFAULT_UPSTREAM_REMOTE" != "$UPSTREAM_REMOTE" ]; then - DEFAULT_UPSTREAM_BRANCHES=`echo \ - ${DEFAULT_UPSTREAM_REMOTE}/master \ - ${DEFAULT_UPSTREAM_REMOTE}/{release,maint}-0.4.1 \ - ${DEFAULT_UPSTREAM_REMOTE}/{release,maint}-0.4.0 \ - ${DEFAULT_UPSTREAM_REMOTE}/{release,maint}-0.3.5 \ - ${DEFAULT_UPSTREAM_REMOTE}/{release,maint}-0.2.9 \ - ` + DEFAULT_UPSTREAM_BRANCHES=$(echo \ + "$DEFAULT_UPSTREAM_REMOTE"/master \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.1 \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.0 \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.3.5 \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.2.9 \ + ) fi -UPSTREAM_BRANCHES=`echo \ - ${UPSTREAM_REMOTE}/master \ - ${UPSTREAM_REMOTE}/{release,maint}-0.4.1 \ - ${UPSTREAM_REMOTE}/{release,maint}-0.4.0 \ - ${UPSTREAM_REMOTE}/{release,maint}-0.3.5 \ - ${UPSTREAM_REMOTE}/{release,maint}-0.2.9 \ - ` +UPSTREAM_BRANCHES=$(echo \ + "$UPSTREAM_REMOTE"/master \ + "$UPSTREAM_REMOTE"/{release,maint}-0.4.1 \ + "$UPSTREAM_REMOTE"/{release,maint}-0.4.0 \ + "$UPSTREAM_REMOTE"/{release,maint}-0.3.5 \ + "$UPSTREAM_REMOTE"/{release,maint}-0.2.9 \ + ) ######################## # Git branches to push # @@ -154,13 +154,13 @@ else # Test branch mode: merge to maint only, and create a new branch for 0.2.9 # # List of branches to push. Ordering is not important. - PUSH_BRANCHES=$(echo \ + PUSH_BRANCHES=" \ ${TEST_BRANCH_PREFIX}_master \ ${TEST_BRANCH_PREFIX}_041 \ ${TEST_BRANCH_PREFIX}_040 \ ${TEST_BRANCH_PREFIX}_035 \ ${TEST_BRANCH_PREFIX}_029 \ - ) + " fi ############### @@ -168,13 +168,13 @@ fi ############### # Skip the test branches that are the same as the upstream branches -if [ "$PUSH_SAME" -eq 0 -a "$TEST_BRANCH_PREFIX" ]; then +if [ "$PUSH_SAME" -eq 0 ] && [ "$TEST_BRANCH_PREFIX" ]; then NEW_PUSH_BRANCHES= for b in $PUSH_BRANCHES; do - PUSH_COMMIT=`git rev-parse "$b"` + PUSH_COMMIT=$(git rev-parse "$b") SKIP_UPSTREAM= for u in $DEFAULT_UPSTREAM_BRANCHES $UPSTREAM_BRANCHES; do - UPSTREAM_COMMIT=`git rev-parse "$u"` + UPSTREAM_COMMIT=$(git rev-parse "$u") if [ "$PUSH_COMMIT" = "$UPSTREAM_COMMIT" ]; then SKIP_UPSTREAM="$u" fi