diff options
Diffstat (limited to 'scripts/git')
| -rwxr-xr-x | scripts/git/git-install-tools.sh | 189 | ||||
| -rwxr-xr-x | scripts/git/git-list-tor-branches.sh | 153 | ||||
| -rwxr-xr-x | scripts/git/git-merge-forward.sh | 373 | ||||
| -rwxr-xr-x | scripts/git/git-pull-all.sh | 208 | ||||
| -rwxr-xr-x | scripts/git/git-push-all.sh | 309 | ||||
| -rwxr-xr-x | scripts/git/git-setup-dirs.sh | 508 | ||||
| -rwxr-xr-x | scripts/git/post-merge.git-hook | 51 | ||||
| -rwxr-xr-x | scripts/git/pre-commit.git-hook | 99 | ||||
| -rwxr-xr-x | scripts/git/pre-push.git-hook | 129 |
9 files changed, 2019 insertions, 0 deletions
diff --git a/scripts/git/git-install-tools.sh b/scripts/git/git-install-tools.sh new file mode 100755 index 0000000000..ef8623a018 --- /dev/null +++ b/scripts/git/git-install-tools.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash + +SCRIPT_NAME=$(basename "$0") +SCRIPTS_DIR=$(dirname "$0") + +TOOL_NAMES=(push-all pull-all merge-forward list-tor-branches) + +function usage() +{ + echo "$SCRIPT_NAME [-h] [-n] [-v] [-f] <all|hooks|tools|aliases>" + echo + echo " flags:" + echo " -h: show this help text" + echo " -n: dry-run" + echo " -v: verbose mode" + echo " -f: force-install even if \$TOR_DEVTOOLS_DIR looks fishy" + echo + echo " modes:" + echo " hooks: install git hooks in this repository." + echo " tools: install scripts in \$TOR_DEVTOOLS_DIR" + echo " aliases: set up global git aliases for git tools in \$TOR_DEVTOOLS_DIR" + echo " all: all of the above." +} + +INSTALL_HOOKS=0 +INSTALL_TOOLS=0 +INSTALL_ALIASES=0 + +DRY_RUN=0 +VERBOSE=0 +FORCE=0 + +while getopts "hnfv" opt; do + case "$opt" in + h) usage + exit 0 + ;; + n) DRY_RUN=1 + ;; + v) VERBOSE=1 + ;; + f) FORCE=1 + ;; + *) echo + usage + exit 1 + ;; + esac +done + +for item in "${@:$OPTIND}"; do + case "$item" in + hooks) INSTALL_HOOKS=1 + ;; + tools) INSTALL_TOOLS=1 + ;; + aliases) INSTALL_ALIASES=1 + ;; + all) INSTALL_HOOKS=1 + INSTALL_TOOLS=1 + INSTALL_ALIASES=1 + ;; + *) echo "Unrecognized mode '$item'" + usage + exit 1 + ;; + esac +done + +if [[ $VERBOSE = 1 ]]; then + function note() + { + echo "$@" + } +else + function note() + { + true + } +fi + +function fail() +{ + echo "$@" 1>&2 + exit 1 +} + +if [[ $INSTALL_HOOKS = 0 && $INSTALL_TOOLS = 0 && $INSTALL_ALIASES = 0 ]]; then + echo "Nothing to do. Try $SCRIPT_NAME -h for a list of commands." + exit 0 +fi + +if [[ $INSTALL_TOOLS = 1 || $INSTALL_ALIASES = 1 ]]; then + if [[ -z "$TOR_DEVTOOLS_DIR" ]] ; then + fail "\$TOR_DEVTOOLS_DIR was not set." + fi + note "Checking whether \$TOR_DEVTOOLS_DIR ($TOR_DEVTOOLS_DIR) is a git repo..." + GITDIR=$(cd "$TOR_DEVTOOLS_DIR" && git rev-parse --git-dir 2>/dev/null) + note "GITDIR is $GITDIR" + if [[ -n "$GITDIR" ]] ; then + cat <<EOF +You have asked me to install to \$TOR_DEVTOOLS_DIR ($TOR_DEVTOOLS_DIR). +That is inside a git repository, so you might not want to install there: +depending on what you pull or push, you might find yourself giving somebody +else write access to your scripts. I think you should just use ~/bin or +something. +EOF + + echo + if [[ "$FORCE" = 1 ]] ; then + echo "I will install anyway, since you said '-f'." + else + echo "I will not install. You can tell me -f if you are really sure." + exit 1 + fi + else + note "It was not." + fi +fi + +if [[ ! -d "$SCRIPTS_DIR" || ! -e "$SCRIPTS_DIR/git-push-all.sh" ]]; then + fail "Couldn't find scripts in '$SCRIPTS_DIR'" +fi + +if [[ $DRY_RUN = 1 ]]; then + echo "** DRY RUN **" + RUN="echo >>" +else + RUN= +fi + +set -e + +# ====================================================================== +if [[ $INSTALL_HOOKS = 1 ]]; then + HOOKS_DIR=$(git rev-parse --git-path hooks) + + note "Looking for hooks directory" + + if [[ -z "$HOOKS_DIR" || ! -d "$HOOKS_DIR" ]]; then + fail "Couldn't find git hooks directory." + fi + + note "Found hooks directory in $HOOKS_DIR" + + note "Installing hooks" + for fn in "$SCRIPTS_DIR"/*.git-hook; do + name=$(basename "$fn") + $RUN install --backup "$fn" "${HOOKS_DIR}/${name%.git-hook}" + done +fi + + +# ====================================================================== +if [[ $INSTALL_TOOLS = 1 ]]; then + note "Installing tools." + note "Looking for \$TOR_DEVTOOLS_DIR ($TOR_DEVTOOLS_DIR)" + + if [[ ! -d "$TOR_DEVTOOLS_DIR" ]]; then + note "Creating directory" + $RUN mkdir -p "$TOR_DEVTOOLS_DIR" + fi + + note "Copying scripts" + for tool in "${TOOL_NAMES[@]}"; do + $RUN install --backup "${SCRIPTS_DIR}/git-${tool}.sh" "${TOR_DEVTOOLS_DIR}/" + done +fi + +# ====================================================================== +if [[ $INSTALL_ALIASES = 1 ]]; then + note "Installing aliases." + note "Looking for \$TOR_DEVTOOLS_DIR ($TOR_DEVTOOLS_DIR)" + + note "Checking for ${TOR_DEVTOOLS_DIR}/git-push-all.sh" + if [[ ! -x "${TOR_DEVTOOLS_DIR}/git-push-all.sh" ]]; then + if [[ $DRY_RUN = 0 ]]; then + fail "Could not find scripts in \$TOR_DEVTOOLS_DIR" + fi + fi + + note "Setting aliases" + for tool in "${TOOL_NAMES[@]}"; do + $RUN git config --global "alias.$tool" \!"${TOR_DEVTOOLS_DIR}/git-${tool}.sh" + done + +fi + +note Done. diff --git a/scripts/git/git-list-tor-branches.sh b/scripts/git/git-list-tor-branches.sh new file mode 100755 index 0000000000..d6b30f064f --- /dev/null +++ b/scripts/git/git-list-tor-branches.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash + +# Script to be used by other git scripts, and provide a single place +# that lists our supported branches. To change which branches are +# supported, look at the end of the file that says 'edit here'. + +SCRIPT_NAME=$(basename "$0") + +function usage() +{ + echo "$SCRIPT_NAME [-h] [-l|-s|-b|-m] [-R]" + echo + echo " arguments:" + echo " -h: show this help text" + echo + echo " -l: list the active tor branches (default)" + echo " -s: list the suffixes to be used with the active tor branches" + echo " -b: write bash code setting WORKTREE to an array of ( branch path ) arrays" + echo " -m: write bash code setting WORKTREE to an array of" + echo " ( branch parent path suffix parent_suffix ) arrays" + echo + echo " -R: omit release branches." +} + +# list : just a list of branch names. +# branch_path : For git-setup-dirs.sh and git-pull-all.sh +# suffix: write a list of suffixes. +# merge: branch, upstream, path, suffix, upstream suffix. +mode="list" +skip_release_branches="no" + +while getopts "hblmsR" opt ; do + case "$opt" in + h) usage + exit 0 + ;; + b) mode="branch_path" + ;; + l) mode="list" + ;; + s) mode="suffix" + ;; + m) mode="merge" + ;; + R) skip_release_branches="yes" + ;; + *) echo "Unknown option" + exit 1 + ;; + esac +done + +all_branch_vars=() + +prev_maint_branch="" +prev_maint_suffix="" + +branch() { + # The name of the branch. (Supplied by caller) Ex: maint-0.4.3 + brname="$1" + + # The name of the branch with no dots. Ex: maint-043 + brname_nodots="${brname//./}" + # The name of the branch with no dots, and _ instead of -. Ex: maint_043 + brname_nodots_uscore="${brname_nodots//-/_}" + # Name to use for a variable to represent the branch. Ex: MAINT_043 + varname="${brname_nodots_uscore^^}" + + is_maint="no" + + # suffix: a suffix to place at the end of branches we generate with respect + # to this branch. Ex: _043 + + # location: where the branch can be found. + + if [[ "$brname" == "master" ]]; then + suffix="_master" + location="\$GIT_PATH/\$TOR_MASTER_NAME" + elif [[ "$brname" =~ ^maint- ]]; then + suffix="_${brname_nodots#maint-}" + location="\$GIT_PATH/\$TOR_WKT_NAME/$brname" + is_maint="yes" + elif [[ "$brname" =~ ^release- ]]; then + suffix="_r${brname_nodots#release-}" + location="\$GIT_PATH/\$TOR_WKT_NAME/$brname" + + if [[ "$skip_release_branches" = "yes" ]]; then + return + fi + else + echo "Unrecognized branch type '${brname}'" >&2 + exit 1 + fi + + all_branch_vars+=("$varname") + + # Now emit the per-branch information + if [[ "$mode" == "branch_path" ]]; then + echo "${varname}=( \"$brname\" \"$location\" )" + elif [[ "$mode" == "merge" ]]; then + echo "${varname}=( \"$brname\" \"$prev_maint_branch\" \"$location\" \"$suffix\" \"$prev_maint_suffix\" )" + elif [[ "$mode" == "list" ]]; then + echo "$brname" + elif [[ "$mode" == "suffix" ]]; then + echo "$suffix" + else + echo "unknown mode $mode" >&2 + exit 1 + fi + + if [[ "$is_maint" == "yes" ]]; then + prev_maint_branch="$brname" + prev_maint_suffix="$suffix" + fi +} + +finish() { + if [[ "$mode" == branch_path ]] || [[ "$mode" == merge ]]; then + echo "WORKTREE=(" + for v in "${all_branch_vars[@]}"; do + echo " ${v}[@]" + done + echo ")" + elif [[ "$mode" == list ]] || [[ "$mode" == suffix ]]; then + # nothing to do + : + else + echo "unknown mode $mode" >&2 + exit 1 + fi +} + +# ============================== +# EDIT HERE +# ============================== +# List of all branches. These must be in order, from oldest to newest, with +# maint before release. + +branch maint-0.3.5 +branch release-0.3.5 + +branch maint-0.4.1 +branch release-0.4.1 + +branch maint-0.4.2 +branch release-0.4.2 + +branch maint-0.4.3 +branch release-0.4.3 + +branch master + +finish diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh new file mode 100755 index 0000000000..7c72f8478d --- /dev/null +++ b/scripts/git/git-merge-forward.sh @@ -0,0 +1,373 @@ +#!/usr/bin/env bash + +SCRIPT_NAME=$(basename "$0") + +function usage() +{ + echo "$SCRIPT_NAME [-h] [-n] [-t <test-branch-prefix> [-u]]" + echo + echo " arguments:" + echo " -h: show this help text" + echo " -n: dry run mode" + echo " (default: run commands)" + echo " -t: test branch mode: create new branches from the commits checked" + echo " out in each maint directory. Call these branches prefix_035," + echo " prefix_040, ... , prefix_master." + echo " (default: merge forward maint-*, release-*, and master)" + echo " -u: in test branch mode, if a prefix_* branch already exists," + echo " skip creating that branch. Use after a merge error, to" + echo " restart the merge forward at the first unmerged branch." + echo " (default: if a prefix_* branch already exists, fail and exit)" + echo + echo " env vars:" + echo " required:" + echo " TOR_FULL_GIT_PATH: where the git repository directories reside." + echo " You must set this env var, we recommend \$HOME/git/" + echo " (default: fail if this env var is not set;" + echo " current: $GIT_PATH)" + echo + echo " optional:" + echo " TOR_MASTER: the name of the directory containing the tor.git clone" + echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " (default: tor; current: $TOR_MASTER_NAME)" + echo " TOR_WKT_NAME: the name of the directory containing the tor" + echo " worktrees. The tor worktrees are:" + echo " \$GIT_PATH/\$TOR_WKT_NAME/{maint-*,release-*}" + echo " (default: tor-wkt; current: $TOR_WKT_NAME)" + echo " we recommend that you set these env vars in your ~/.profile" +} + +################# +# Configuration # +################# + +# Don't change this configuration - set the env vars in your .profile + +# Where are all those git repositories? +GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} +# The tor master git repository directory from which all the worktree have +# been created. +TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} +# The worktrees location (directory). +TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} + +########################## +# Git branches to manage # +########################## + +# The branches and worktrees need to be modified when there is a new branch, +# and when an old branch is no longer supported. + +# Configuration of the branches that needs merging. The values are in order: +# (0) current maint/release branch name +# (1) previous maint/release name to merge into (0) +# (only used in merge forward mode) +# (2) Full path of the git worktree +# (3) current branch suffix +# (maint branches only, only used in test branch mode) +# (4) previous test branch suffix to merge into (3) +# (maint branches only, only used in test branch mode) +# +# Merge forward example: +# $ cd <PATH/TO/WORKTREE> (2) +# $ git checkout maint-0.3.5 (0) +# $ git pull +# $ git merge maint-0.3.4 (1) +# +# Test branch example: +# $ cd <PATH/TO/WORKTREE> (2) +# $ git checkout -b ticket99999_035 (3) +# $ git checkout maint-0.3.5 (0) +# $ git pull +# $ git checkout ticket99999_035 +# $ git merge maint-0.3.5 +# $ git merge ticket99999_034 (4) +# +# First set of arrays are the maint-* branch and then the release-* branch. +# New arrays need to be in the WORKTREE= array else they aren't considered. +# +# Only used in test branch mode +# We create a test branch for the earliest maint branch. +# But it's the earliest maint branch, so we don't merge forward into it. +# Since we don't merge forward into it, the second and fifth items must be +# blank (""). + +# origin that will be used to fetch the updates. All the worktrees are created +# from that repository. +ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" + +####################### +# Argument processing # +####################### + +# Controlled by the -n option. The dry run option will just output the command +# that would have been executed for each worktree. +DRY_RUN=0 + +# Controlled by the -t <test-branch-prefix> option. The test branch base +# name option makes git-merge-forward.sh create new test branches: +# <tbbn>_035, <tbbn>_040, ... , <tbbn>_master, and merge forward. +TEST_BRANCH_PREFIX= + +# Controlled by the -u option. The use existing option checks for existing +# branches with the <test-branch-prefix>, and checks them out, rather than +# creating a new branch. +USE_EXISTING=0 + +while getopts "hnt:u" opt; do + case "$opt" in + h) usage + exit 0 + ;; + n) DRY_RUN=1 + echo " *** DRY RUN MODE ***" + ;; + t) TEST_BRANCH_PREFIX="$OPTARG" + echo " *** CREATING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***" + ;; + u) USE_EXISTING=1 + echo " *** USE EXISTING TEST BRANCHES MODE ***" + ;; + *) + echo + usage + exit 1 + ;; + esac +done + +########################### +# Git worktrees to manage # +########################### + +set -e +if [ -z "$TEST_BRANCH_PREFIX" ]; then + # maint/release merge mode + eval "$(git-list-tor-branches.sh -m)" + # Remove first element: we don't merge forward into it. + WORKTREE=( "${WORKTREE[@]:1}" ) +else + eval "$(git-list-tor-branches.sh -m -R)" +fi +set +e + +COUNT=${#WORKTREE[@]} + +############# +# Constants # +############# + +# Control characters +CNRM=$'\x1b[0;0m' # Clear color + +# Bright color +BGRN=$'\x1b[1;32m' +BBLU=$'\x1b[1;34m' +BRED=$'\x1b[1;31m' +BYEL=$'\x1b[1;33m' +IWTH=$'\x1b[3;37m' + +# Strings for the pretty print. +MARKER="${BBLU}[${BGRN}+${BBLU}]${CNRM}" +SUCCESS="${BGRN}success${CNRM}" +FAILED="${BRED}failed${CNRM}" + +#################### +# Helper functions # +#################### + +# Validate the given returned value (error code), print success or failed. The +# second argument is the error output in case of failure, it is printed out. +# On failure, this function exits. +function validate_ret +{ + if [ "$1" -eq 0 ]; then + printf "%s\\n" "$SUCCESS" + else + printf "%s\\n" "$FAILED" + printf " %s" "$2" + exit 1 + fi +} + +# Switch to the given branch name. +function switch_branch +{ + local cmd="git checkout '$1'" + printf " %s Switching branch to %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Checkout a new branch with the given branch name. +function new_branch +{ + local cmd="git checkout -b '$1'" + printf " %s Creating new branch %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Switch to an existing branch, or checkout a new branch with the given +# branch name. +function switch_or_new_branch +{ + local cmd="git rev-parse --verify '$1'" + if [ $DRY_RUN -eq 0 ]; then + # Call switch_branch if there is a branch, or new_branch if there is not + msg=$( eval "$cmd" 2>&1 ) + RET=$? + if [ $RET -eq 0 ]; then + # Branch: (commit id) + switch_branch "$1" + elif [ $RET -eq 128 ]; then + # Not a branch: "fatal: Needed a single revision" + new_branch "$1" + else + # Unexpected return value + validate_ret $RET "$msg" + fi + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}, then depending on the result:" + switch_branch "$1" + new_branch "$1" + fi +} + +# Pull the given branch name. +function pull_branch +{ + local cmd="git pull" + printf " %s Pulling branch %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Merge the given branch name ($1) into the current branch ($2). +function merge_branch +{ + local cmd="git merge --no-edit '$1'" + printf " %s Merging branch %s into %s..." "$MARKER" "$1" "$2" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Merge origin/(branch name) into the current branch. +function merge_branch_origin +{ + local cmd="git merge --ff-only 'origin/$1'" + printf " %s Merging branch origin/%s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Go into the worktree repository. +function goto_repo +{ + if [ ! -d "$1" ]; then + echo " $1: Not found. Stopping." + exit 1 + fi + cd "$1" || exit +} + +# Fetch the origin. No arguments. +function fetch_origin +{ + local cmd="git fetch origin" + printf " %s Fetching origin..." "$MARKER" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +############### +# Entry point # +############### + +# First, fetch the origin. +goto_repo "$ORIGIN_PATH" +fetch_origin + +# Go over all configured worktree. +for ((i=0; i<COUNT; i++)); do + current=${!WORKTREE[$i]:0:1} + previous=${!WORKTREE[$i]:1:1} + repo_path=${!WORKTREE[$i]:2:1} + # default to merge forward mode + test_current= + test_previous= + target_current="$current" + target_previous="$previous" + if [ "$TEST_BRANCH_PREFIX" ]; then + test_current_suffix=${!WORKTREE[$i]:3:1} + test_current=${TEST_BRANCH_PREFIX}${test_current_suffix} + # the current test branch, if present, or maint/release branch, if not + target_current="$test_current" + test_previous_suffix=${!WORKTREE[$i]:4:1} + if [ "$test_previous_suffix" ]; then + test_previous=${TEST_BRANCH_PREFIX}${test_previous_suffix} + # the previous test branch, if present, or maint/release branch, if not + target_previous="$test_previous" + fi + fi + + printf "%s Handling branch \\n" "$MARKER" "${BYEL}$target_current${CNRM}" + + # Go into the worktree to start merging. + goto_repo "$repo_path" + if [ "$test_current" ]; then + if [ $USE_EXISTING -eq 0 ]; then + # Create a test branch from the currently checked-out branch/commit + # Fail if it already exists + new_branch "$test_current" + else + # Switch if it exists, or create if it does not + switch_or_new_branch "$test_current" + fi + fi + # Checkout the current maint/release branch + switch_branch "$current" + # Update the current maint/release branch with an origin merge to get the + # latest updates + merge_branch_origin "$current" + if [ "$test_current" ]; then + # Checkout the test branch + switch_branch "$test_current" + # Merge the updated maint branch into the test branch + merge_branch "$current" "$test_current" + fi + # Merge the previous branch into the target branch + # Merge Forward Example: + # merge maint-0.3.5 into maint-0.4.0. + # Test Branch Example: + # merge bug99999_035 into bug99999_040. + # Skip the merge if the previous branch does not exist + # (there's nothing to merge forward into the oldest test branch) + if [ "$target_previous" ]; then + merge_branch "$target_previous" "$target_current" + fi +done diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh new file mode 100755 index 0000000000..7f82eda296 --- /dev/null +++ b/scripts/git/git-pull-all.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash + +SCRIPT_NAME=$(basename "$0") + +usage() +{ + echo "$SCRIPT_NAME [-h] [-n]" + echo + echo " arguments:" + echo " -h: show this help text" + echo " -n: dry run mode" + echo " (default: run commands)" + echo + echo " env vars:" + echo " required:" + echo " TOR_FULL_GIT_PATH: where the git repository directories reside." + echo " You must set this env var, we recommend \$HOME/git/" + echo " (default: fail if this env var is not set;" + echo " current: $GIT_PATH)" + echo + echo " optional:" + echo " TOR_MASTER: the name of the directory containing the tor.git clone" + echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " (default: tor; current: $TOR_MASTER_NAME)" + echo " TOR_WKT_NAME: the name of the directory containing the tor" + echo " worktrees. The tor worktrees are:" + echo " \$GIT_PATH/\$TOR_WKT_NAME/{maint-*,release-*}" + echo " (default: tor-wkt; current: $TOR_WKT_NAME)" + echo " we recommend that you set these env vars in your ~/.profile" +} + +################# +# Configuration # +################# + +# Don't change this configuration - set the env vars in your .profile + +# Where are all those git repositories? +GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} +# The tor master git repository directory from which all the worktree have +# been created. +TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} +# The worktrees location (directory). +TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} + +########################## +# Git branches to manage # +########################## + +set -e +eval "$(git-list-tor-branches.sh -b)" +set +e + +# The master branch path has to be the main repository thus contains the +# origin that will be used to fetch the updates. All the worktrees are created +# from that repository. +ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" + +COUNT=${#WORKTREE[@]} + +####################### +# Argument processing # +####################### + +# Controlled by the -n option. The dry run option will just output the command +# that would have been executed for each worktree. +DRY_RUN=0 + +while getopts "hn" opt; do + case "$opt" in + h) usage + exit 0 + ;; + n) DRY_RUN=1 + echo " *** DRY DRUN MODE ***" + ;; + *) + echo + usage + exit 1 + ;; + esac +done + +############# +# Constants # +############# + +# Control characters +CNRM=$'\x1b[0;0m' # Clear color + +# Bright color +BGRN=$'\x1b[1;32m' +BBLU=$'\x1b[1;34m' +BRED=$'\x1b[1;31m' +BYEL=$'\x1b[1;33m' +IWTH=$'\x1b[3;37m' + +# Strings for the pretty print. +MARKER="${BBLU}[${BGRN}+${BBLU}]${CNRM}" +SUCCESS="${BGRN}ok${CNRM}" +FAILED="${BRED}failed${CNRM}" + +#################### +# Helper functions # +#################### + +# Validate the given returned value (error code), print success or failed. The +# second argument is the error output in case of failure, it is printed out. +# On failure, this function exits. +function validate_ret +{ + if [ "$1" -eq 0 ]; then + printf "%s\\n" "$SUCCESS" + else + printf "%s\\n" "$FAILED" + printf " %s" "$2" + exit 1 + fi +} + +# Switch to the given branch name. +function switch_branch +{ + local cmd="git checkout $1" + printf " %s Switching branch to %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Pull the given branch name. +function merge_branch +{ + local cmd="git merge --ff-only origin/$1" + printf " %s Merging branch origin/%s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Go into the worktree repository. +function goto_repo +{ + if [ ! -d "$1" ]; then + echo " $1: Not found. Stopping." + exit 1 + fi + cd "$1" || exit +} + +# Fetch the origin. No arguments. +function fetch_origin +{ + local cmd="git fetch origin" + printf " %s Fetching origin..." "$MARKER" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Fetch tor-github pull requests. No arguments. +function fetch_tor_github +{ + local cmd="git fetch tor-github" + printf " %s Fetching tor-github..." "$MARKER" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +############### +# Entry point # +############### + +# First, fetch tor-github. +goto_repo "$ORIGIN_PATH" +fetch_tor_github + +# Then, fetch the origin. +fetch_origin + +# Go over all configured worktree. +for ((i=0; i<COUNT; i++)); do + current=${!WORKTREE[$i]:0:1} + repo_path=${!WORKTREE[$i]:1:1} + + printf "%s Handling branch %s\\n" "$MARKER" "${BYEL}$current${CNRM}" + + # Go into the worktree to start merging. + goto_repo "$repo_path" + # Checkout the current branch + switch_branch "$current" + # Update the current branch by merging the origin to get the latest. + merge_branch "$current" +done diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh new file mode 100755 index 0000000000..558ea8d01c --- /dev/null +++ b/scripts/git/git-push-all.sh @@ -0,0 +1,309 @@ +#!/usr/bin/env bash + +SCRIPT_NAME=$(basename "$0") + +function usage() +{ + if [ "$TOR_PUSH_SAME" ]; then + CURRENT_PUSH_SAME="push" + else + CURRENT_PUSH_SAME="skip" + fi + + echo "$SCRIPT_NAME [-h] [-r <remote-name> [-t <test-branch-prefix>]] [-s]" + # The next line looks misaligned, but it lines up in the output + echo " [-- [-n] [--no-atomic] <git push options>]" + echo + echo " arguments:" + echo " -h: show this help text" + echo " -n: dry run mode" + echo " (default: run commands)" + echo " -r: push to remote-name, rather than the default upstream remote." + echo " (default: $DEFAULT_UPSTREAM_REMOTE, current: $UPSTREAM_REMOTE)" + echo " -t: test branch mode: push test branches to remote-name. Pushes" + echo " branches prefix_035, prefix_040, ... , prefix_master." + echo " (default: push maint-*, release-*, and master)" + echo " -s: push branches whose tips match upstream maint, release, or" + echo " master branches. The default is to skip these branches," + echo " because they do not contain any new code. Use -s to test for" + echo " CI environment failures, using code that previously passed CI." + echo " (default: skip; current: $CURRENT_PUSH_SAME matching branches)" + echo " --: pass further arguments to git push." + echo " All unrecognised arguments are passed to git push, but complex" + echo " arguments before -- may be mangled by getopt." + echo " (default: git push --atomic, current: $GIT_PUSH)" + echo + echo " env vars:" + echo " optional:" + echo " TOR_GIT_PUSH_PATH: change to this directory before pushing." + echo " (default: if \$TOR_FULL_GIT_PATH is set," + echo " use \$TOR_FULL_GIT_PATH/\$TOR_MASTER;" + echo " Otherwise, use the current directory for pushes;" + echo " current: $TOR_GIT_PUSH_PATH)" + echo " TOR_FULL_GIT_PATH: where the git repository directories reside." + echo " We recommend using \$HOME/git/." + echo " (default: use the current directory for pushes;" + echo " current: $TOR_FULL_GIT_PATH)" + echo " TOR_MASTER: the name of the directory containing the tor.git clone" + echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " (default: tor; current: $TOR_MASTER_NAME)" + echo + echo " TOR_UPSTREAM_REMOTE_NAME: the default upstream remote." + echo " Overridden by -r." + echo " (default: upstream; current: $UPSTREAM_REMOTE)" + echo " TOR_GIT_PUSH: the git push command and default arguments." + echo " Overridden by <git push options> after --." + echo " (default: git push --atomic; current: $GIT_PUSH)" + echo " TOR_PUSH_SAME: push branches whose tips match upstream maint," + echo " release, or master branches. Inverted by -s." + echo " (default: skip; current: $CURRENT_PUSH_SAME matching branches)" + echo " TOR_PUSH_DELAY: pushes the master and maint branches separately," + echo " so that CI runs in a sensible order." + echo " (default: push all branches immediately; current: $PUSH_DELAY)" + echo " we recommend that you set these env vars in your ~/.profile" +} + +set -e + +################# +# Configuration # +################# + +# Don't change this configuration - set the env vars in your .profile +# +# The tor master git repository directory from which all the worktree have +# been created. +TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} +# Which directory do we push from? +if [ "$TOR_FULL_GIT_PATH" ]; then + TOR_GIT_PUSH_PATH=${TOR_GIT_PUSH_PATH:-"$TOR_FULL_GIT_PATH/$TOR_MASTER_NAME"} +fi +# git push command and default arguments +GIT_PUSH=${TOR_GIT_PUSH:-"git push --atomic"} +# The upstream remote which git.torproject.org/tor.git points to. +DEFAULT_UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} +# Push to a different upstream remote using -r <remote-name> +UPSTREAM_REMOTE=${DEFAULT_UPSTREAM_REMOTE} +# Add a delay between pushes, so CI runs on the most important branches first +PUSH_DELAY=${TOR_PUSH_DELAY:-0} +# Push (1) or skip (0) test branches that are the same as an upstream +# maint/master branch. Push if you are testing that the CI environment still +# works on old code, skip if you are testing new code in the branch. +# Default: skip unchanged branches. +# Inverted by the -s option. +PUSH_SAME=${TOR_PUSH_SAME:-0} + +####################### +# Argument processing # +####################### + +# Controlled by the -t <test-branch-prefix> option. The test branch prefix +# option makes git-merge-forward.sh create new test branches: +# <tbp>_035, <tbp>_040, ... , <tbp>_master, and merge each branch forward into +# the next one. +TEST_BRANCH_PREFIX= + +while getopts ":hr:st:" opt; do + case "$opt" in + h) usage + exit 0 + ;; + r) UPSTREAM_REMOTE="$OPTARG" + echo " *** PUSHING TO REMOTE: ${UPSTREAM_REMOTE} ***" + shift + shift + OPTIND=$((OPTIND - 2)) + ;; + s) PUSH_SAME=$((! PUSH_SAME)) + if [ "$PUSH_SAME" -eq 0 ]; then + echo " *** SKIPPING UNCHANGED TEST BRANCHES ***" + else + echo " *** PUSHING UNCHANGED TEST BRANCHES ***" + fi + shift + OPTIND=$((OPTIND - 1)) + ;; + t) TEST_BRANCH_PREFIX="$OPTARG" + echo " *** PUSHING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***" + shift + shift + OPTIND=$((OPTIND - 2)) + ;; + *) + # Make git push handle the option + # This might mangle options with spaces, use -- for complex options + GIT_PUSH="$GIT_PUSH $1" + shift + OPTIND=$((OPTIND - 1)) + ;; + esac +done + +# getopts doesn't allow "-" as an option character, +# so we have to handle -- manually +if [ "$1" = "--" ]; then + shift +fi + +if [ "$TEST_BRANCH_PREFIX" ]; then + if [ "$UPSTREAM_REMOTE" = "$DEFAULT_UPSTREAM_REMOTE" ]; then + echo "Pushing test branches ${TEST_BRANCH_PREFIX}_nnn to " \ + "the default remote $DEFAULT_UPSTREAM_REMOTE is not allowed." + echo + usage + exit 1 + fi +fi + +if [ "$TOR_GIT_PUSH_PATH" ]; then + echo "Changing to $TOR_GIT_PUSH_PATH before pushing" + cd "$TOR_GIT_PUSH_PATH" +else + echo "Pushing from the current directory" +fi + +echo "Calling $GIT_PUSH" "$@" "<branches>" + +################################ +# Git upstream remote branches # +################################ + +set -e +DEFAULT_UPSTREAM_BRANCHES= +if [ "$DEFAULT_UPSTREAM_REMOTE" != "$UPSTREAM_REMOTE" ]; then + for br in $(git-list-tor-branches.sh -l); do + DEFAULT_UPSTREAM_BRANCHES="${DEFAULT_UPSTREAM_BRANCHES} ${DEFAULT_UPSTREAM_REMOTE}/${br}" + done +fi + +UPSTREAM_BRANCHES= +for br in $(git-list-tor-branches.sh -l); do + UPSTREAM_BRANCHES="${UPSTREAM_BRANCHES} ${UPSTREAM_REMOTE}/${br}" +done + +######################## +# Git branches to push # +######################## + +if [ -z "$TEST_BRANCH_PREFIX" ]; then + + # maint/release push mode: push all branches. + # + # List of branches to push. Ordering is not important. + PUSH_BRANCHES="$(git-list-tor-branches.sh -l)" +else + + # Test branch push mode: push test branches, based on each maint branch. + # + # List of branches to push. Ordering is not important. + PUSH_BRANCHES="" + for suffix in $(git-list-tor-branches.sh -s -R); do + PUSH_BRANCHES="${PUSH_BRANCHES} ${TEST_BRANCH_PREFIX}${suffix}" + done +fi + +set +e + +############### +# Entry point # +############### + +if [ "$TEST_BRANCH_PREFIX" ]; then + # Skip the test branches that are the same as the default or current + # upstream branches (they have already been tested) + UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES $DEFAULT_UPSTREAM_BRANCHES" +else + # Skip the local maint-*, release-*, master branches that are the same as the + # current upstream branches, but ignore the default upstream + # (we want to update a non-default remote, even if it matches the default) + UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES" +fi + +# Skip branches that match the relevant upstream(s) +if [ "$PUSH_SAME" -eq 0 ]; then + NEW_PUSH_BRANCHES= + for b in $PUSH_BRANCHES; do + PUSH_COMMIT=$(git rev-parse "$b") + SKIP_UPSTREAM= + for u in $UPSTREAM_SKIP_SAME_AS; do + # Skip the branch check on error + UPSTREAM_COMMIT=$(git rev-parse "$u" 2>/dev/null) || continue + if [ "$PUSH_COMMIT" = "$UPSTREAM_COMMIT" ]; then + SKIP_UPSTREAM="$u" + fi + done + if [ "$SKIP_UPSTREAM" ]; then + printf "Skipping unchanged: %s matching remote: %s\\n" \ + "$b" "$SKIP_UPSTREAM" + else + if [ "$NEW_PUSH_BRANCHES" ]; then + NEW_PUSH_BRANCHES="${NEW_PUSH_BRANCHES} ${b}" + else + NEW_PUSH_BRANCHES="${b}" + fi + fi + done + PUSH_BRANCHES=${NEW_PUSH_BRANCHES} +fi + +if [ ! "$PUSH_BRANCHES" ]; then + echo "No branches to push!" + # We expect the rest of the script to run without errors, even if there + # are no branches +fi + +if [ "$PUSH_DELAY" -le 0 ]; then + echo "Pushing $PUSH_BRANCHES" + # We know that there are no spaces in any branch within $PUSH_BRANCHES, so + # it is safe to use it unquoted. (This also applies to the other shellcheck + # exceptions below.) + # + # Push all the branches at the same time + # shellcheck disable=SC2086 + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $PUSH_BRANCHES +else + # Push the branches in optimal CI order, with a delay between each push + PUSH_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | sort -V) + MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep master) \ + || true # Skipped master branch + if [ -z "$TEST_BRANCH_PREFIX" ]; then + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep maint) \ + || true # Skipped all maint branches + RELEASE_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep release | \ + tr "\\n" " ") || true # Skipped all release branches + else + # Actually test branches based on maint branches + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v master) \ + || true # Skipped all maint test branches + # No release branches + RELEASE_BRANCHES= + fi + if [ "$MASTER_BRANCH" ] || [ "$MAINT_BRANCHES" ] \ + || [ "$RELEASE_BRANCHES" ]; then + printf "Pushing with %ss delays, so CI runs in this order:\\n" \ + "$PUSH_DELAY" + if [ "$MASTER_BRANCH" ]; then + printf "%s\\n" "$MASTER_BRANCH" + fi + if [ "$MAINT_BRANCHES" ]; then + printf "%s\\n" "$MAINT_BRANCHES" + fi + if [ "$RELEASE_BRANCHES" ]; then + printf "%s\\n" "$RELEASE_BRANCHES" + fi + fi + # shellcheck disable=SC2086 + for b in $MASTER_BRANCH $MAINT_BRANCHES; do + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$b" + # If we are pushing more than one branch, delay. + # In the unlikely scenario where we are pushing maint without master, + # or maint without release, there may be an extra delay + if [ "$MAINT_BRANCHES" ] || [ "$RELEASE_BRANCHES" ]; then + sleep "$PUSH_DELAY" + fi + done + if [ "$RELEASE_BRANCHES" ]; then + # shellcheck disable=SC2086 + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $RELEASE_BRANCHES + fi +fi diff --git a/scripts/git/git-setup-dirs.sh b/scripts/git/git-setup-dirs.sh new file mode 100755 index 0000000000..1f61eb8b83 --- /dev/null +++ b/scripts/git/git-setup-dirs.sh @@ -0,0 +1,508 @@ +#!/usr/bin/env bash + +SCRIPT_NAME=$(basename "$0") + +function usage() +{ + echo "$SCRIPT_NAME [-h] [-n] [-u]" + echo + echo " arguments:" + echo " -h: show this help text" + echo " -n: dry run mode" + echo " (default: run commands)" + echo " -u: if a directory or worktree already exists, use it" + echo " (default: fail and exit on existing directories)" + echo + echo " env vars:" + echo " required:" + echo " TOR_FULL_GIT_PATH: where the git repository directories reside." + echo " You must set this env var, we recommend \$HOME/git/" + echo " (default: fail if this env var is not set;" + echo " current: $GIT_PATH)" + echo + echo " optional:" + echo " TOR_MASTER: the name of the directory containing the tor.git clone" + echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " (default: tor; current: $TOR_MASTER_NAME)" + echo " TOR_WKT_NAME: the name of the directory containing the tor" + echo " worktrees. The tor worktrees are:" + echo " \$GIT_PATH/\$TOR_WKT_NAME/{maint-*,release-*}" + echo " (default: tor-wkt; current: $TOR_WKT_NAME)" + echo " TOR_GIT_ORIGIN_PULL: the origin remote pull URL." + echo " (current: $GIT_ORIGIN_PULL)" + echo " TOR_GIT_ORIGIN_PUSH: the origin remote push URL" + echo " (current: $GIT_ORIGIN_PUSH)" + echo " TOR_UPSTREAM_REMOTE_NAME: the default upstream remote." + echo " If \$TOR_UPSTREAM_REMOTE_NAME is not 'origin', we have a" + echo " separate upstream remote, and we don't push to origin." + echo " (default: $DEFAULT_UPSTREAM_REMOTE)" + echo " TOR_GITHUB_PULL: the tor-github remote pull URL" + echo " (current: $GITHUB_PULL)" + echo " TOR_GITHUB_PUSH: the tor-github remote push URL" + echo " (current: $GITHUB_PUSH)" + echo " TOR_EXTRA_CLONE_ARGS: extra arguments to git clone" + echo " (current: $TOR_EXTRA_CLONE_ARGS)" + echo " TOR_EXTRA_REMOTE_NAME: the name of an extra remote" + echo " This remote is not pulled by this script or git-pull-all.sh." + echo " This remote is not pushed by git-push-all.sh." + echo " (current: $TOR_EXTRA_REMOTE_NAME)" + echo " TOR_EXTRA_REMOTE_PULL: the extra remote pull URL." + echo " (current: $TOR_EXTRA_REMOTE_PULL)" + echo " TOR_EXTRA_REMOTE_PUSH: the extra remote push URL" + echo " (current: $TOR_EXTRA_REMOTE_PUSH)" + echo " we recommend that you set these env vars in your ~/.profile" +} + +################# +# Configuration # +################# + +# Don't change this configuration - set the env vars in your .profile + +# Where are all those git repositories? +GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} +# The tor master git repository directory from which all the worktree have +# been created. +TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} +# The worktrees location (directory). +TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} + +# Origin repositories +GIT_ORIGIN_PULL=${TOR_GIT_ORIGIN_PULL:-"https://git.torproject.org/tor.git"} +GIT_ORIGIN_PUSH=${TOR_GIT_ORIGIN_PUSH:-"git@git-rw.torproject.org:tor.git"} +# The upstream remote which git.torproject.org/tor.git points to. +DEFAULT_UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} +# Copy the URLs from origin +GIT_UPSTREAM_PULL="$GIT_ORIGIN_PULL" +GIT_UPSTREAM_PUSH="$GIT_ORIGIN_PUSH" +# And avoid pushing to origin if we have an upstream +if [ "$DEFAULT_UPSTREAM_REMOTE" != "origin" ]; then + GIT_ORIGIN_PUSH="No pushes to origin, if there is an upstream" +fi +# GitHub repositories +GITHUB_PULL=${TOR_GITHUB_PULL:-"https://github.com/torproject/tor.git"} +GITHUB_PUSH=${TOR_GITHUB_PUSH:-"No_Pushing_To_GitHub"} + +########################## +# Git branches to manage # +########################## + +# The branches and worktrees need to be modified when there is a new branch, +# and when an old branch is no longer supported. + +set -e +eval "$(git-list-tor-branches.sh -b)" +set +e + +# The master branch path has to be the main repository thus contains the +# origin that will be used to fetch the updates. All the worktrees are created +# from that repository. +ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" + +####################### +# Argument processing # +####################### + +# Controlled by the -n option. The dry run option will just output the command +# that would have been executed for each worktree. +DRY_RUN=0 + +# Controlled by the -s option. The use existing option checks for existing +# directories, and re-uses them, rather than creating a new directory. +USE_EXISTING=0 +USE_EXISTING_HINT="Use existing: '$SCRIPT_NAME -u'." + +while getopts "hnu" opt; do + case "$opt" in + h) usage + exit 0 + ;; + n) DRY_RUN=1 + echo " *** DRY RUN MODE ***" + ;; + u) USE_EXISTING=1 + echo " *** USE EXISTING DIRECTORIES MODE ***" + ;; + *) + echo + usage + exit 1 + ;; + esac +done + +########################### +# Git worktrees to manage # +########################### + +COUNT=${#WORKTREE[@]} + +############# +# Constants # +############# + +# Control characters +CNRM=$'\x1b[0;0m' # Clear color + +# Bright color +BGRN=$'\x1b[1;32m' +BBLU=$'\x1b[1;34m' +BRED=$'\x1b[1;31m' +BYEL=$'\x1b[1;33m' +IWTH=$'\x1b[3;37m' + +# Strings for the pretty print. +MARKER="${BBLU}[${BGRN}+${BBLU}]${CNRM}" +SUCCESS="${BGRN}success${CNRM}" +SKIPPED="${BYEL}skipped${CNRM}" +FAILED="${BRED}failed${CNRM}" + +#################### +# Helper functions # +#################### + +# Validate the given returned value (error code), print success or failed. The +# second argument is the error output in case of failure, it is printed out. +# On failure, this function exits. +function validate_ret +{ + if [ "$1" -eq 0 ]; then + printf "%s\\n" "$SUCCESS" + else + printf "%s\\n" "$FAILED" + printf " %s\\n" "$2" + exit 1 + fi +} + +# Validate the given returned value (error code), print success, skipped, or +# failed. If $USE_EXISTING is 0, fail on error, otherwise, skip on error. +# The second argument is the error output in case of failure, it is printed +# out. On failure, this function exits. +function validate_ret_skip +{ + if [ "$1" -ne 0 ]; then + if [ "$USE_EXISTING" -eq "0" ]; then + # Fail and exit with error + validate_ret "$1" "$2 $USE_EXISTING_HINT" + else + printf "%s\\n" "$SKIPPED" + printf " %s\\n" "${IWTH}$2${CNRM}" + # Tell the caller to skip the rest of the function + return 0 + fi + fi + # Tell the caller to continue + return 1 +} + +# Create a directory, and any missing enclosing directories. +# If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip. +function make_directory +{ + local cmd="mkdir -p '$1'" + printf " %s Creating directory %s..." "$MARKER" "$1" + local check_cmd="[ ! -d '$1' ]" + msg=$( eval "$check_cmd" 2>&1 ) + if validate_ret_skip $? "Directory already exists."; then + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Create a symlink from the first argument to the second argument +# If the link already exists: fail if $USE_EXISTING is 0, otherwise skip. +function make_symlink +{ + local cmd="ln -s '$1' '$2'" + printf " %s Creating symlink from %s to %s..." "$MARKER" "$1" "$2" + local check_cmd="[ ! -e '$2' ]" + msg=$( eval "$check_cmd" 2>&1 ) + if validate_ret_skip $? "File already exists."; then + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Go into the directory or repository, even if $DRY_RUN is non-zero. +# If the directory does not exist, fail and log an error. +# Otherwise, silently succeed. +function goto_dir +{ + if ! cd "$1" 1>/dev/null 2>/dev/null ; then + printf " %s Changing to directory %s..." "$MARKER" "$1" + validate_ret 1 "$1: Not found. Stopping." + fi +} + +# Clone a repository into a directory. +# If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip. +function clone_repo +{ + local cmd="git clone $TOR_EXTRA_CLONE_ARGS '$1' '$2'" + printf " %s Cloning %s into %s..." "$MARKER" "$1" "$2" + local check_cmd="[ ! -d '$2' ]" + msg=$( eval "$check_cmd" 2>&1 ) + if validate_ret_skip $? "Directory already exists."; then + # If we skip the clone, we need to do a fetch + goto_dir "$ORIGIN_PATH" + fetch_remote "origin" + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Add a remote by name and URL. +# If the remote already exists: fail if $USE_EXISTING is 0, otherwise skip. +function add_remote +{ + local cmd="git remote add '$1' '$2'" + printf " %s Adding remote %s at %s..." "$MARKER" "$1" "$2" + local check_cmd="git remote get-url '$1'" + msg=$( eval "$check_cmd" 2>&1 ) + ret=$? + # We don't want a remote, so we invert the exit status + if validate_ret_skip $(( ! ret )) \ + "Remote already exists for $1 at $msg."; then + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Set a remote's push URL by name and URL. +function set_remote_push +{ + local cmd="git remote set-url --push '$1' '$2'" + printf " %s Setting remote %s push URL to '%s'..." "$MARKER" "$1" "$2" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Fetch a remote by name. +function fetch_remote +{ + local cmd="git fetch '$1'" + printf " %s Fetching %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Replace the fetch configs for a remote with config if they match a pattern. +function replace_fetch_config +{ + local cmd="git config --replace-all remote.'$1'.fetch '$2' '$3'" + printf " %s Replacing %s fetch configs for '%s'..." \ + "$MARKER" "$1" "$3" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Set up the tor-github PR config, so tor-github/pr/NNNN/head points to GitHub +# PR NNNN. In some repositories, "/head" is optional. +function set_tor_github_pr_fetch_config +{ + # Standard branches + replace_fetch_config tor-github \ + "+refs/heads/*:refs/remotes/tor-github/*" \ + "refs/heads" + # PRs + replace_fetch_config "tor-github" \ + "+refs/pull/*:refs/remotes/tor-github/pr/*" \ + "refs/pull.*pr" +} + +# Add a new worktree for branch at path. +# If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip. +function add_worktree +{ + local cmd="git worktree add '$2' '$1'" + printf " %s Adding worktree for %s at %s..." "$MARKER" "$1" "$2" + local check_cmd="[ ! -d '$2' ]" + msg=$( eval "$check_cmd" 2>&1 ) + if validate_ret_skip $? "Directory already exists."; then + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Switch to the given branch name. +# If the branch does not exist: fail. +function switch_branch +{ + local cmd="git checkout '$1'" + printf " %s Switching branch to %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Checkout a new branch with the given branch name. +# If the branch already exists: fail if $USE_EXISTING is 0, otherwise skip. +function new_branch +{ + local cmd="git checkout -b '$1'" + printf " %s Creating new branch %s..." "$MARKER" "$1" + local check_cmd="git branch --list '$1'" + msg=$( eval "$check_cmd" 2>&1 ) + if validate_ret_skip $? "Branch already exists."; then + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Switch to an existing branch, or checkout a new branch with the given +# branch name. +function switch_or_new_branch +{ + local cmd="git rev-parse --verify '$1'" + if [ $DRY_RUN -eq 0 ]; then + # Call switch_branch if there is a branch, or new_branch if there is not + msg=$( eval "$cmd" 2>&1 ) + RET=$? + if [ $RET -eq 0 ]; then + # Branch: (commit id) + switch_branch "$1" + elif [ $RET -eq 128 ]; then + # Not a branch: "fatal: Needed a single revision" + new_branch "$1" + else + # Unexpected return value + validate_ret $RET "$msg" + fi + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}, then depending on the result:" + switch_branch "$1" + new_branch "$1" + fi +} + +# Set the upstream for branch to upstream. +function set_upstream +{ + # Note the argument order is swapped + local cmd="git branch --set-upstream-to='$2' '$1'" + printf " %s Setting upstream for %s to %s..." "$MARKER" "$1" "$2" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +############### +# Entry point # +############### + +printf "%s Setting up the repository and remote %s\\n" "$MARKER" \ + "${BYEL}origin${CNRM}" +# First, fetch the origin. +ORIGIN_PARENT=$(dirname "$ORIGIN_PATH") +make_directory "$ORIGIN_PARENT" +# This is just cd with an error check +goto_dir "$ORIGIN_PARENT" + +# clone repository / origin remote +clone_repo "$GIT_ORIGIN_PULL" "$TOR_MASTER_NAME" +goto_dir "$ORIGIN_PATH" +set_remote_push "origin" "$GIT_ORIGIN_PUSH" + +# upstream remote, if different to origin +if [ "$DEFAULT_UPSTREAM_REMOTE" != "origin" ]; then + printf "%s Setting up remote %s\\n" "$MARKER" \ + "${BYEL}$DEFAULT_UPSTREAM_REMOTE${CNRM}" + add_remote "$DEFAULT_UPSTREAM_REMOTE" "$GIT_UPSTREAM_PULL" + set_remote_push "$DEFAULT_UPSTREAM_REMOTE" "$GIT_UPSTREAM_PUSH" + fetch_remote "$DEFAULT_UPSTREAM_REMOTE" +fi + +# GitHub remote +printf "%s Setting up remote %s\\n" "$MARKER" "${BYEL}tor-github${CNRM}" +# Add remote +add_remote "tor-github" "$GITHUB_PULL" +set_remote_push "tor-github" "$GITHUB_PUSH" +# Add custom fetch for PRs +set_tor_github_pr_fetch_config +# Now fetch them all +fetch_remote "tor-github" + +# Extra remote +if [ "$TOR_EXTRA_REMOTE_NAME" ]; then + printf "%s Setting up remote %s\\n" "$MARKER" \ + "${BYEL}$TOR_EXTRA_REMOTE_NAME${CNRM}" + # Add remote + add_remote "$TOR_EXTRA_REMOTE_NAME" "$TOR_EXTRA_REMOTE_PULL" + set_remote_push "$TOR_EXTRA_REMOTE_NAME" "$TOR_EXTRA_REMOTE_PUSH" + # But leave it to the user to decide if they want to fetch it + #fetch_remote "$TOR_EXTRA_REMOTE_NAME" +fi + +# Go over all configured worktree. +for ((i=0; i<COUNT; i++)); do + branch=${!WORKTREE[$i]:0:1} + repo_path=${!WORKTREE[$i]:1:1} + + printf "%s Handling branch %s\\n" "$MARKER" "${BYEL}$branch${CNRM}" + # We cloned the repository, and master is the default branch + if [ "$branch" = "master" ]; then + if [ "$TOR_MASTER_NAME" != "master" ]; then + # Set up a master link in the worktree directory + make_symlink "$repo_path" "$GIT_PATH/$TOR_WKT_NAME/master" + fi + else + # git makes worktree directories if they don't exist + add_worktree "origin/$branch" "$repo_path" + fi + goto_dir "$repo_path" + switch_or_new_branch "$branch" + set_upstream "$branch" "origin/$branch" +done + +echo +echo "Remember to copy the git hooks from tor/scripts/git/*.git-hook to" +echo "$ORIGIN_PATH/.git/hooks/*" diff --git a/scripts/git/post-merge.git-hook b/scripts/git/post-merge.git-hook new file mode 100755 index 0000000000..eae4f999e7 --- /dev/null +++ b/scripts/git/post-merge.git-hook @@ -0,0 +1,51 @@ +#!/bin/sh + +# This is post-merge git hook script to check for changes in: +# * git hook scripts +# * helper scripts for using git efficiently. +# If any changes are detected, a diff of them is printed. +# +# To install this script, copy it to .git/hooks/post-merge in local copy of +# tor git repo and make sure it has permission to execute. + +git_toplevel=$(git rev-parse --show-toplevel) + +check_for_diffs() { + installed="$git_toplevel/.git/hooks/$1" + latest="$git_toplevel/scripts/git/$1.git-hook" + + if [ -e "$installed" ] + then + if ! cmp "$installed" "$latest" >/dev/null 2>&1 + then + echo "ATTENTION: $1 hook has changed:" + echo "===============================" + diff -u "$installed" "$latest" + fi + fi +} + +check_for_script_update() { + fullpath="$1" + + if ! git diff ORIG_HEAD HEAD --exit-code -- "$fullpath" >/dev/null + then + echo "ATTENTION: $1 has changed:" + git --no-pager diff ORIG_HEAD HEAD -- "$fullpath" + fi +} + +cur_branch=$(git rev-parse --abbrev-ref HEAD) +if [ "$cur_branch" != "master" ]; then + echo "post-merge: Not a master branch. Skipping." + exit 0 +fi + +check_for_diffs "pre-push" +check_for_diffs "pre-commit" +check_for_diffs "post-merge" + +for file in "$git_toplevel"/scripts/git/* ; do + check_for_script_update "$file" +done + diff --git a/scripts/git/pre-commit.git-hook b/scripts/git/pre-commit.git-hook new file mode 100755 index 0000000000..f630a242bd --- /dev/null +++ b/scripts/git/pre-commit.git-hook @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# +# To install this script, copy it to .git/hooks/pre-commit in local copy of +# tor git repo and make sure it has permission to execute. +# +# This is pre-commit git hook script that prevents commiting your changeset if +# it fails our code formatting, changelog entry formatting, module include +# rules, or best practices tracker. + +workdir=$(git rev-parse --show-toplevel) + +cd "$workdir" || exit 1 + +set -e + +if [ $# -eq 0 ]; then + # When called in pre-commit, check the files modified in this commit + CHECK_FILTER="git diff --cached --name-only --diff-filter=ACMR" + # Use the appropriate owned tor source list to filter the changed files + + # This is the layout in 0.3.5 and later. + + # Keep these lists consistent: + # - OWNED_TOR_C_FILES in Makefile.am + # - CHECK_FILES in pre-commit.git-hook and pre-push.git-hook + # - try_parse in check_cocci_parse.sh + CHECK_FILES="$($CHECK_FILTER \ + src/lib/*/*.[ch] \ + src/core/*/*.[ch] \ + src/feature/*/*.[ch] \ + src/app/*/*.[ch] \ + src/test/*.[ch] \ + src/test/*/*.[ch] \ + src/tools/*.[ch] \ + )" +else + # When called in pre-push, concatenate the argument array + # Fails on special characters in file names + CHECK_FILES="$*" +fi + +## General File Checks + +if [ -n "$(ls ./changes/)" ]; then + python scripts/maint/lintChanges.py ./changes/* +fi + +if [ -e scripts/maint/checkShellScripts.sh ]; then + scripts/maint/checkShellScripts.sh +fi + +# Always run the practracker unit tests +PT_DIR=scripts/maint/practracker + +if [ -e "${PT_DIR}/test_practracker.sh" ]; then + "${PT_DIR}/test_practracker.sh" +fi + +if [ -e scripts/maint/checkSpaceTest.sh ]; then + scripts/maint/checkSpaceTest.sh +fi + +if [ ! "$CHECK_FILES" ]; then + echo "No modified tor-owned source files, skipping further checks" + exit 0 +fi + +## Owned Source File Checks + +printf "Modified tor-owned source files:\\n%s\\n" "$CHECK_FILES" + +# We want word splitting here, because file names are space separated +# shellcheck disable=SC2086 +perl scripts/maint/checkSpace.pl -C \ + $CHECK_FILES + +if test -e scripts/maint/practracker/includes.py; then + python scripts/maint/practracker/includes.py +fi + +# Only call practracker if ${PT_DIR}/.enable_practracker_in_hooks exists +# We do this check so that we can enable practracker in hooks in master, and +# disable it on maint branches +if [ -e "${PT_DIR}/practracker.py" ]; then + if [ -e "${PT_DIR}/.enable_practracker_in_hooks" ]; then + python3 "${PT_DIR}/practracker.py" "$workdir" + fi +fi + +if [ -e scripts/coccinelle/check_cocci_parse.sh ]; then + + # Run a verbose cocci parse check on the changed files + # (spatch is slow, so we don't want to check all the files.) + # + # We want word splitting here, because file names are space separated + # shellcheck disable=SC2086 + VERBOSE=1 scripts/coccinelle/check_cocci_parse.sh \ + $CHECK_FILES +fi diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook new file mode 100755 index 0000000000..efa45b9860 --- /dev/null +++ b/scripts/git/pre-push.git-hook @@ -0,0 +1,129 @@ +#!/usr/bin/env bash + +# git pre-push hook script to: +# 0) Call the pre-commit hook, if it is available +# 1) prevent "fixup!" and "squash!" commit from ending up in master, release-* +# or maint-* +# 2) Disallow pushing branches other than master, release-* +# and maint-* to origin (e.g. gitweb.torproject.org) +# +# To install this script, copy it into .git/hooks/pre-push path in your +# local copy of git repository. Make sure it has permission to execute. +# Furthermore, make sure that TOR_UPSTREAM_REMOTE_NAME environment +# variable is set to local name of git remote that corresponds to upstream +# repository on e.g. git.torproject.org. +# +# The following sample script was used as starting point: +# https://github.com/git/git/blob/master/templates/hooks--pre-push.sample + +# Are you adding a new check to the git hooks? +# - Common checks belong in the pre-commit hook +# - Push-only checks belong in the pre-push hook + +echo "Running pre-push hook" + +z40=0000000000000000000000000000000000000000 + +upstream_name=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} + +# The working directory +workdir=$(git rev-parse --show-toplevel) +# The .git directory +# If $workdir is a worktree, then $gitdir is not $workdir/.git +gitdir=$(git rev-parse --git-dir) + +cd "$workdir" || exit 1 + +remote="$1" +remote_name=$(git remote --verbose | grep "$2" | awk '{print $1}' | head -n 1) + + +ref_is_upstream_branch() { + if [ "$1" == "refs/heads/master" ] || + [[ "$1" == refs/heads/release-* ]] || + [[ "$1" == refs/heads/maint-* ]]; then + return 1 + fi +} + +# shellcheck disable=SC2034 +while read -r local_ref local_sha remote_ref remote_sha +do + if [ "$local_sha" = $z40 ]; then + # Handle delete + : + else + if [ "$remote_sha" = $z40 ]; then + # New branch, examine commits not in master + range="master...$local_sha" + else + # Update to existing branch, examine new commits + range="$remote_sha..$local_sha" + fi + + # Call the pre-commit hook for the common checks, if it is executable + pre_commit=${gitdir}/hooks/pre-commit + if [ -x "$pre_commit" ]; then + # Only check the files newly modified in this branch + CHECK_FILTER="git diff --name-only --diff-filter=ACMR $range" + # Use the appropriate owned tor source list to filter the changed + # files + # This is the layout in 0.3.5 + # Keep these lists consistent: + # - OWNED_TOR_C_FILES in Makefile.am + # - CHECK_FILES in pre-commit.git-hook and pre-push.git-hook + # - try_parse in check_cocci_parse.sh + CHECK_FILES="$($CHECK_FILTER \ + src/lib/*/*.[ch] \ + src/core/*/*.[ch] \ + src/feature/*/*.[ch] \ + src/app/*/*.[ch] \ + src/test/*.[ch] \ + src/test/*/*.[ch] \ + src/tools/*.[ch] \ + )" + + # We want word splitting here, because file names are space + # separated + # shellcheck disable=SC2086 + if ! "$pre_commit" $CHECK_FILES ; then + exit 1 + fi + fi + + if [[ "$remote_name" != "$upstream_name" ]]; then + echo "Not pushing to upstream - refraining from further checks" + continue + fi + + if (ref_is_upstream_branch "$local_ref" == 0 || + ref_is_upstream_branch "$remote_ref" == 0) && + [ "$local_ref" != "$remote_ref" ]; then + if [ "$remote" == "origin" ]; then + echo >&2 "Not pushing: $local_ref to $remote_ref" + echo >&2 "If you really want to push this, use --no-verify." + exit 1 + else + continue + fi + fi + + # Check for fixup! commit + commit=$(git rev-list -n 1 --grep '^fixup!' "$range") + if [ -n "$commit" ]; then + echo >&2 "Found fixup! commit in $local_ref, not pushing" + echo >&2 "If you really want to push this, use --no-verify." + exit 1 + fi + + # Check for squash! commit + commit=$(git rev-list -n 1 --grep '^squash!' "$range") + if [ -n "$commit" ]; then + echo >&2 "Found squash! commit in $local_ref, not pushing" + echo >&2 "If you really want to push this, use --no-verify." + exit 1 + fi + fi +done + +exit 0 |