diff --git a/changes/ticket31314 b/changes/ticket31314 new file mode 100644 index 0000000000..7ce96e96cf --- /dev/null +++ b/changes/ticket31314 @@ -0,0 +1,18 @@ + o Minor features (git scripts): + - Add a -t argument to git-merge-forward.sh and + git-push-all.sh, which makes these scripts create, merge forward, and + push test branches. Closes ticket 31314. + - Add a -r argument to git-push-all.sh, so the script can + push test branches to a personal remote. Closes ticket 31314. + - Add a -u argument to git-merge-forward.sh, so that the script can re-use + existing test branches after a merge failure and fix. + Closes ticket 31314. + - Add a TOR_GIT_PUSH env var, which sets the default git push command and + arguments for git-push-all.sh. Closes ticket 31314. + - Add a "--" command-line argument, to + separate git-push-all.sh script arguments from arguments that are passed + through to git push. Closes ticket 31314. + - Skip pushing test branches that are the same as a remote + maint/release/master branch in git-push-all.sh by default. Add a -s + argument, so git-push-all.sh can push all test branches. + Closes ticket 31314. diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index ba29983284..cbd2f3c3bd 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -1,9 +1,23 @@ #!/usr/bin/env bash -############################## -# Configuration (change me!) # -############################## +# Usage: git-merge-forward.sh -n -t -u +# arguments: +# -n: dry run mode +# -t: test branch mode: create new branches from the commits checked +# out in each maint directory. Call these branches prefix_029, +# prefix_035, ... , prefix_master. +# -u: in test branch mode, if a prefix_* branch exists, skip creating +# that branch. Use after a merge error, to restart the merge +# forward at the first unmerged branch. +# env vars: +# See the Configuration section for env vars and their default values. +################# +# Configuration # +################# + +# Don't change this configuration - set the env vars in your .profile +# # The general setup that is suggested here is: # # GIT_PATH = /home//git/ @@ -21,27 +35,55 @@ TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} -######################### -# End of configuration. # -######################### +########################## +# Git branches to manage # +########################## + +# The branches and worktrees need to be modified when there is a new branch, +# and when an old branch is no longer supported. # Configuration of the branches that needs merging. The values are in order: -# (1) Branch name that we merge onto. -# (2) Branch name to merge from. In other words, this is merge into (1) -# (3) Full path of the git worktree. +# (0) current maint/release branch name +# (1) previous maint/release name to merge into (0) +# (only used in merge forward mode) +# (2) Full path of the git worktree +# (3) current branch suffix +# (maint branches only, only used in test branch mode) +# (4) previous test branch suffix to merge into (3) +# (maint branches only, only used in test branch mode) # -# As an example: -# $ cd (3) -# $ git checkout maint-0.3.5 (1) +# Merge forward example: +# $ cd (2) +# $ git checkout maint-0.3.5 (0) # $ git pull -# $ git merge maint-0.3.4 (2) +# $ git merge maint-0.3.4 (1) +# +# Test branch example: +# $ cd (2) +# $ git checkout -b ticket99999_035 (3) +# $ git checkout maint-0.3.5 (0) +# $ git pull +# $ git checkout ticket99999_035 +# $ git merge maint-0.3.5 +# $ git merge ticket99999_034 (4) # # First set of arrays are the maint-* branch and then the release-* branch. # New arrays need to be in the WORKTREE= array else they aren't considered. -MAINT_035=( "maint-0.3.5" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) -MAINT_040=( "maint-0.4.0" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) -MAINT_041=( "maint-0.4.1" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" ) -MAINT_MASTER=( "master" "maint-0.4.1" "$GIT_PATH/$TOR_MASTER_NAME" ) +# +# Only used in test branch mode +# There is no previous branch to merge forward, so the second and fifth items +# must be blank ("") +MAINT_029_TB=( "maint-0.2.9" "" "$GIT_PATH/$TOR_WKT_NAME/maint-0.2.9" \ + "_029" "") +# Used in maint/release merge and test branch modes +MAINT_035=( "maint-0.3.5" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" \ + "_035" "_029") +MAINT_040=( "maint-0.4.0" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" \ + "_040" "_035") +MAINT_041=( "maint-0.4.1" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" \ + "_041" "_040") +MAINT_MASTER=( "master" "maint-0.4.1" "$GIT_PATH/$TOR_MASTER_NAME" \ + "_master" "_041") RELEASE_029=( "release-0.2.9" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) RELEASE_035=( "release-0.3.5" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) @@ -55,6 +97,7 @@ ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" # SC2034 -- shellcheck thinks that these are unused. We know better. ACTUALLY_THESE_ARE_USED=< option. The test branch base +# name option makes git-merge-forward.sh create new test branches: +# _029, _035, ... , _master, and merge forward. +TEST_BRANCH_PREFIX= + +# Controlled by the -u option. The use existing option checks for existing +# branches with the , and checks them out, rather than +# creating a new branch. +USE_EXISTING=0 + +while getopts "nt:u" opt; do + case "$opt" in + n) DRY_RUN=1 + echo " *** DRY RUN MODE ***" + ;; + t) TEST_BRANCH_PREFIX="$OPTARG" + echo " *** CREATING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***" + ;; + u) USE_EXISTING=1 + echo " *** USE EXISTING TEST BRANCHES MODE ***" + ;; + *) + exit 1 + ;; + esac +done + +########################### +# Git worktrees to manage # +########################### + +if [ -z "$TEST_BRANCH_PREFIX" ]; then + + # maint/release merge mode + # + # List of all worktrees to work on. All defined above. Ordering is important. + # Always the maint-* branch BEFORE then the release-*. + WORKTREE=( + RELEASE_029[@] + + MAINT_035[@] + RELEASE_035[@] + + MAINT_040[@] + RELEASE_040[@] + + MAINT_041[@] + RELEASE_041[@] + + MAINT_MASTER[@] + ) + +else + + # Test branch mode: merge to maint only, and create a new branch for 0.2.9 + WORKTREE=( + MAINT_029_TB[@] + + MAINT_035[@] + + MAINT_040[@] + + MAINT_041[@] + + MAINT_MASTER[@] + ) + +fi + +COUNT=${#WORKTREE[@]} + +############# +# Constants # +############# + # Control characters CNRM=$'\x1b[0;0m' # Clear color @@ -127,7 +227,7 @@ function validate_ret # Switch to the given branch name. function switch_branch { - local cmd="git checkout $1" + local cmd="git checkout '$1'" printf " %s Switching branch to %s..." "$MARKER" "$1" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) @@ -137,6 +237,45 @@ function switch_branch fi } +# Checkout a new branch with the given branch name. +function new_branch +{ + local cmd="git checkout -b '$1'" + printf " %s Creating new branch %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Switch to an existing branch, or checkout a new branch with the given +# branch name. +function switch_or_new_branch +{ + local cmd="git rev-parse --verify '$1'" + if [ $DRY_RUN -eq 0 ]; then + # Call switch_branch if there is a branch, or new_branch if there is not + msg=$( eval "$cmd" 2>&1 ) + RET=$? + if [ $RET -eq 0 ]; then + # Branch: (commit id) + switch_branch "$1" + elif [ $RET -eq 128 ]; then + # Not a branch: "fatal: Needed a single revision" + new_branch "$1" + else + # Unexpected return value + validate_ret $RET "$msg" + fi + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}, then depending on the result:" + switch_branch "$1" + new_branch "$1" + fi +} + # Pull the given branch name. function pull_branch { @@ -150,10 +289,10 @@ function pull_branch fi } -# Merge the given branch name ($2) into the current branch ($1). +# Merge the given branch name ($1) into the current branch ($2). function merge_branch { - local cmd="git merge --no-edit $1" + local cmd="git merge --no-edit '$1'" printf " %s Merging branch %s into %s..." "$MARKER" "$1" "$2" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) @@ -166,7 +305,7 @@ function merge_branch # Pull the given branch name. function merge_branch_origin { - local cmd="git merge --ff-only origin/$1" + local cmd="git merge --ff-only 'origin/$1'" printf " %s Merging branch origin/%s..." "$MARKER" "$1" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) @@ -203,16 +342,6 @@ function fetch_origin # Entry point # ############### -while getopts "n" opt; do - case "$opt" in - n) DRY_RUN=1 - echo " *** DRY DRUN MODE ***" - ;; - *) - ;; - esac -done - # First, fetch the origin. goto_repo "$ORIGIN_PATH" fetch_origin @@ -222,15 +351,57 @@ for ((i=0; i -r -s +# -- +# arguments: +# -t: test branch mode: Push test branches, rather than maint and +# release branches. Pushes the branches called prefix_029, +# prefix_035, ... , prefix_master. +# -r: push to remote-name, rather than $TOR_UPSTREAM_REMOTE_NAME. +# -s: push branches whose tips match upstream maint, release, or +# master branches. The default is to skip these branches. Use +# -s when testing for CI environment failures with old code. +# --: pass any other arguments to git, rather than the script. +# env vars: +# TOR_GIT_PUSH: the git push command and arguments +# TOR_UPSTREAM_REMOTE_NAME: the default upstream, overridden by -r +# TOR_PUSH_DELAY: pushes the master and maint branches separately, +# so that CI runs in a sensible order. +# TOR_PUSH_SAME: push branches whose tips match upstream maint, +# release, or master branches. Inverted by -s. +# See the Configuration section for env var default values. +# git-opts: +# --no-atomic --dry-run (and any other git push option) set -e +################# +# Configuration # +################# + +# Don't change this configuration - set the env vars in your .profile +# +# git push command and default arguments +GIT_PUSH=${TOR_GIT_PUSH:-"git push --atomic"} # The upstream remote which git.torproject.org/tor.git points to. -UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} +DEFAULT_UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} +# Push to a different upstream remote using -r +UPSTREAM_REMOTE=${DEFAULT_UPSTREAM_REMOTE} # Add a delay between pushes, so CI runs on the most important branches first PUSH_DELAY=${TOR_PUSH_DELAY:-0} +# Push (1) or skip (0) test branches that are the same as an upstream +# maint/master branch. Push if you are testing that the CI environment still +# works on old code, skip if you are testing new code in the branch. +# Default: skip unchanged branches. +# Inverted by the -s option. +PUSH_SAME=${TOR_PUSH_SAME:-0} + +####################### +# Argument processing # +####################### + +# Controlled by the -t option. The test branch base +# name option makes git-merge-forward.sh create new test branches: +# _029, _035, ... , _master, and merge forward. +TEST_BRANCH_PREFIX= + +while getopts ":r:st:" opt; do + case "$opt" in + r) UPSTREAM_REMOTE="$OPTARG" + echo " *** PUSHING TO REMOTE: ${UPSTREAM_REMOTE} ***" + shift + shift + OPTIND=$((OPTIND - 2)) + ;; + s) PUSH_SAME=$((! PUSH_SAME)) + if [ "$PUSH_SAME" -eq 0 ]; then + echo " *** SKIPPING UNCHANGED TEST BRANCHES ***" + else + echo " *** PUSHING UNCHANGED TEST BRANCHES ***" + fi + shift + OPTIND=$((OPTIND - 1)) + ;; + t) TEST_BRANCH_PREFIX="$OPTARG" + echo " *** PUSHING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***" + shift + shift + OPTIND=$((OPTIND - 2)) + ;; + *) + # Assume we're done with script arguments, + # and git push will handle the option + break + ;; + esac +done + +# getopts doesn't allow "-" as an option character, +# so we have to handle -- manually +if [ "$1" = "--" ]; then + shift +fi + +echo "Calling $GIT_PUSH" "$@" "" + +if [ "$TEST_BRANCH_PREFIX" ]; then + if [ "$UPSTREAM_REMOTE" = "${TOR_UPSTREAM_REMOTE_NAME:-upstream}" ]; then + echo "Pushing test branches ${TEST_BRANCH_PREFIX}_nnn to " \ + "$UPSTREAM_REMOTE is not allowed." + echo "Usage: $0 -r -t " + exit 1 + fi +fi + +################################ +# Git upstream remote branches # +################################ + +DEFAULT_UPSTREAM_BRANCHES= +if [ "$DEFAULT_UPSTREAM_REMOTE" != "$UPSTREAM_REMOTE" ]; then + DEFAULT_UPSTREAM_BRANCHES=$(echo \ + "$DEFAULT_UPSTREAM_REMOTE"/master \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.1 \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.0 \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.3.5 \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.2.9 \ + ) +fi + +UPSTREAM_BRANCHES=$(echo \ + "$UPSTREAM_REMOTE"/master \ + "$UPSTREAM_REMOTE"/{release,maint}-0.4.1 \ + "$UPSTREAM_REMOTE"/{release,maint}-0.4.0 \ + "$UPSTREAM_REMOTE"/{release,maint}-0.3.5 \ + "$UPSTREAM_REMOTE"/{release,maint}-0.2.9 \ + ) + +######################## +# Git branches to push # +######################## PUSH_BRANCHES=$(echo \ master \ @@ -23,29 +137,98 @@ PUSH_BRANCHES=$(echo \ {release,maint}-0.2.9 \ ) +if [ -z "$TEST_BRANCH_PREFIX" ]; then + + # maint/release push mode + # + # List of branches to push. Ordering is not important. + PUSH_BRANCHES=$(echo \ + master \ + {release,maint}-0.4.1 \ + {release,maint}-0.4.0 \ + {release,maint}-0.3.5 \ + {release,maint}-0.2.9 \ + ) +else + + # Test branch mode: merge to maint only, and create a new branch for 0.2.9 + # + # List of branches to push. Ordering is not important. + PUSH_BRANCHES=" \ + ${TEST_BRANCH_PREFIX}_master \ + ${TEST_BRANCH_PREFIX}_041 \ + ${TEST_BRANCH_PREFIX}_040 \ + ${TEST_BRANCH_PREFIX}_035 \ + ${TEST_BRANCH_PREFIX}_029 \ + " +fi + +############### +# Entry point # +############### + +# Skip the test branches that are the same as the upstream branches +if [ "$PUSH_SAME" -eq 0 ] && [ "$TEST_BRANCH_PREFIX" ]; then + NEW_PUSH_BRANCHES= + for b in $PUSH_BRANCHES; do + PUSH_COMMIT=$(git rev-parse "$b") + SKIP_UPSTREAM= + for u in $DEFAULT_UPSTREAM_BRANCHES $UPSTREAM_BRANCHES; do + UPSTREAM_COMMIT=$(git rev-parse "$u") + if [ "$PUSH_COMMIT" = "$UPSTREAM_COMMIT" ]; then + SKIP_UPSTREAM="$u" + fi + done + if [ "$SKIP_UPSTREAM" ]; then + printf "Skipping unchanged: %s remote: %s\n" \ + "$b" "$SKIP_UPSTREAM" + else + if [ "$NEW_PUSH_BRANCHES" ]; then + NEW_PUSH_BRANCHES="${NEW_PUSH_BRANCHES} ${b}" + else + NEW_PUSH_BRANCHES="${b}" + fi + fi + done + PUSH_BRANCHES=${NEW_PUSH_BRANCHES} +fi + if [ "$PUSH_DELAY" -le 0 ]; then echo "Pushing $PUSH_BRANCHES" # We know that there are no spaces in any branch within $PUSH_BRANCHES, so # it is safe to use it unquoted. (This also applies to the other shellcheck # exceptions below.) # + # Push all the branches at the same time # shellcheck disable=SC2086 - git push --atomic "$@" "$UPSTREAM_REMOTE" $PUSH_BRANCHES + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $PUSH_BRANCHES else + # Push the branches in optimal CI order, with a delay between each push PUSH_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | sort -V) MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep master) - MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep maint) - RELEASE_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep release | \ - tr "\n" " ") - printf "Pushing with %ss delays, so CI runs in this order:\n%s\n%s\n%s\n" \ - "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES" "$RELEASE_BRANCHES" - git push "$@" "$UPSTREAM_REMOTE" "$MASTER_BRANCH" + if [ -z "$TEST_BRANCH_PREFIX" ]; then + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep maint) + RELEASE_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep release | \ + tr "\n" " ") + printf "Pushing with %ss delays, so CI runs in this order:\n%s\n%s\n%s\n" \ + "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES" "$RELEASE_BRANCHES" + else + # Actually test branches based on maint branches + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep -v master) + printf "Pushing with %ss delays, so CI runs in this order:\n%s\n%s\n" \ + "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES" + # No release branches + RELEASE_BRANCHES= + fi + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$MASTER_BRANCH" sleep "$PUSH_DELAY" # shellcheck disable=SC2086 for b in $MAINT_BRANCHES; do - git push "$@" "$UPSTREAM_REMOTE" $b + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$b" sleep "$PUSH_DELAY" done - # shellcheck disable=SC2086 - git push --atomic "$@" "$UPSTREAM_REMOTE" $RELEASE_BRANCHES + if [ "$RELEASE_BRANCHES" ]; then + # shellcheck disable=SC2086 + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $RELEASE_BRANCHES + fi fi