path: root/scripts/vcs.sh

#!/bin/sh
# This script wraps both bzr and git and provides a uniform interface
# to both in such a way that python code which invokes this script doesn't
# need to have very much specific knowledge of either bzr or git.

# Note: All informational logging should be routed to STDERR. STDOUT becomes
# the return value of this script in the calling python code, and must be kept
# clean for parsing. Unless the exit code is nonzero, then STDOUT should be
# a concise error message. See vcs.py for how STDOUT/STDERR are treated.

# Fuzz startup time a bit since this gets forked in parallel a lot.
sleep "$(shuf --input-range 0-10 --head-count 1)"

set -eu

TWENTY_MINUTES="1200"
CODEDIR=$(dirname "$(readlink --canonicalize "$0")")
CACHE_DIR="/tmp/bileto/$VCS-cache"
WORKSPACE="$(pwd)"
WORKPARENT="$WORKSPACE/$SERIES/$SOURCE"
WORKDIR="$WORKPARENT/$SOURCE"
SWIFT_CONTAINER="bileto-$TICKETID"
[ -d "$WORKDIR" ] && cd "$WORKDIR" || true
IFS='
'

. $CODEDIR/funcs.sh


escape() {
    echo "$1" | perl -lpe 's/([^-_a-z0-9:~])/sprintf("%%%02X", ord($1))/gie'
}

if [ $# -gt 0 ]; then
    BRANCH="$1"
    CACHE_BASE="$(escape "$BRANCH")"
    CACHE="$CACHE_DIR/$CACHE_BASE"
    CACHE_SOURCE="$WORKPARENT/$CACHE_BASE"
    NO_COMMIT_MESSAGE="Failed to commit $BRANCH. You must supply either a Commit Message on your MP, or a custom debian/changelog entry."
    [ "$VCS" != "git" ] || CACHE="$CACHE_DIR/$(escape "$TARGET")"
fi


mkdir --parents "$CACHE_DIR" "$WORKPARENT" 1>&2 || die "Failed to create working directory."


#####################
# Utility Functions #
#####################

die() {
    echo "$SERIES/$SOURCE:" "$@" 2>&1
    exit 1
}

die_gently() {
    echo "$SERIES/$SOURCE:" "$@" 1>&2
    exit 0
}

parsechangelog() {
    (cd "$1" && dpkg-parsechangelog --show-field "$2"     \
             || die "Failed to parse changelog in $1")
}

force_changelog_version() {
    loudly sed --in-place --expression "1s/.*/$SOURCE ($VERSION) $SERIES; urgency=medium/g" debian/changelog || die "Failed to set changelog version to $VERSION."
    loudly sed --in-place --expression "s/^version:.*/version: $VERSION/" snapcraft.yaml || true
}

gpg_key() {
    # Sort gpg secret keys by creation date and pick the newest one.
    gpg --list-secret-keys --with-colons | awk -F: '/^sec/ {print $6 "\t" $5}' | sort | tail -1 | cut --field 2
}

download_from_dsc() {
    loudly dget --quiet --allow-unauthenticated --download-only "$1" || die "Failed to download DSC file $1"
    DSC_FILE="$(basename "$1")"
    loudly gpg --keyserver "keyserver.ubuntu.com" --keyserver-options "auto-key-retrieve" --verify "$DSC_FILE" || die "Failed to verify DSC file $DSC_FILE"
}

debdiff_wrapper() {
    # Disable gpgv because we already verified the DSCs manually.
    ln --symbolic --force /bin/true $HOME/gpgv
    debdiff_stderr="$(mktemp debdiff-stderr-XXX)"
    (export PATH=$HOME:$PATH && set -x && debdiff --diffstat "$1" "$2" 1>"$3" 2>"$debdiff_stderr") || true
    cat "$debdiff_stderr" 1>&2
    if egrep --ignore-case --quiet 'error|fail' "$debdiff_stderr"; then
        die "debdiff failed: see log for details."
    fi
}

die_if_timed_out() {
    # shellcheck disable=SC2046
    if [ $(ps --pid "$$" --format "etimes=") -gt "$TWENTY_MINUTES" ]; then
        die "Timed out."
    fi
}

get_from_primary() {
    printf "Waiting for %s to finish " "$1" 1>&2
    until [ -f "$1/../done" ]; do
        die_if_timed_out
        printf "." 1>&2
        sleep 10
    done
    printf " done!\n" 1>&2
    loudly cp --archive "$1" "$WORKDIR" || die "Failed to copy source tree from primary."
    cd "$WORKDIR"
}

run_hook() {
    scripts="$(dirname "$(readlink --canonicalize "$0")")"
    export HOOK="$1"
    chmod 711 "$WORKSPACE"  # Needed so unity8 hook script can cd into ./po/
    (cd "$WORKDIR" && sudo --preserve-env /bin/sh $scripts/run_hook.sh || die "Failed to run hook $HOOK.")
}

####################
# Generic Commands #
####################

do_append_changelog() {
    loudly dch --multimaint-merge --release-heuristic changelog --newversion "$VERSION" "$MESSAGE" || die "Failed to add changelog message."
}

do_get_secondary() {
    get_from_primary "$WORKSPACE/$PRIMARY/$SOURCE/$SOURCE"
    orig_version="$(parsechangelog "$WORKDIR" Version)"
    VERSION="$(echo "$orig_version" | sed "s/\+\([[:digit:]]\{2\}\.\)\{2\}\([[:digit:]]\{8\}\)/+$SERIES_VERSION.\2/")"
    force_changelog_version
    run_hook pre_release_hook
    loudly touch "$WORKPARENT/done"
}

do_get_nongles() {
    nongles="$(echo $SOURCE | sed -n 's/-gles$//p')"
    get_from_primary "$WORKSPACE/$SERIES/$nongles/$nongles"
    run_hook convert_to_gles
    new_name="$(parsechangelog "$WORKDIR" Source)"
    [ "$new_name" = "$SOURCE" ] || die "Found $new_name in debian/changelog but expected $SOURCE"
    loudly touch "$WORKPARENT/done"
}

do_revert() {
    download_from_dsc "$DSC_URL"
    loudly dpkg-source -x "$(basename "$DSC_URL")" "$WORKDIR"
    cd "$WORKDIR"
    loudly dch --newversion "$VERSION" "Reverting to $REAL_VERSION due to regression."
    loudly dch --release --force-distribution --distribution "$SERIES" ''
    loudly touch "$WORKPARENT/done"
}

do_build() {
    upstream="$(dpkg-parsechangelog --show-field Version | sed -e 's/^[^:]\+://' -e 's/-[^-]\+$//')"
    orig_tar="../${SOURCE}_${upstream}.orig.tar.gz"
    loudly tar --create --auto-compress --file "$orig_tar" \
        --exclude='.bzr*' --exclude='.git*' --exclude="debian" . || die "Failed to create orig.tar."

    # -sa: Force inclusion of original source.
    # -d:  Do not check build dependencies and conflicts.
    # -S:  Only build source package, no binaries.
    # -i:  Ignore bzr/git files from generated diff (can only be specified once).
    # -I:  Ignore VCS files from generated tarball, plus .bzr-builddeb which isn't included in default -I
    # -nc: Do not run `./debian/rules clean`
    # -k:  Sign .changes with this key.
    # --changes-option=-DLaunchpad-Notify-Changed-By=yes: Sends upload failure emails to person listed in changelog, which is the person who ran the bileto job.
    loudly dpkg-buildpackage -sa -d -S -i'(^|/)(\.bzr|\.git)' -I -I'.bzr*' -nc -k"$(gpg_key)" --force-sign --changes-option=-DLaunchpad-Notify-Changed-By=yes || die "Failed to build source package."
}

do_upload() {
    loudly dput ppa:$PPA_TEAM/$DISTRO/$PPA_NAME $WORKPARENT/${SOURCE}_*.changes || die "Failed to upload package."
}

do_diff_prefetch() {
    cd "$WORKPARENT"
    our_dsc="$(echo ${SOURCE}_*.dsc)"
    if [ ! -f "$our_dsc" ]; then
        [ -z "$DSC_URL" ] || download_from_dsc "$DSC_URL"
    fi
}

_build_packaging_diff() {
    printf " * This diff only shows the debian/ directory! It is not a full content diff!\n\n"
    [ -n "$DSC_URL" ] || printf " * Source package must be preNEWed by an archive admin because it has never been released in Ubuntu before.\n\n"
    [ -z "$COMPONENT" ] || printf " * Source package is a %s component.\n\n" "$COMPONENT"
    packages="$(sed -n 's/^[+-]Package: //p' <"$2" | sort | uniq --unique)"
    [ -z "$packages" ] || printf " * Please consult an archive admin about adding or removing these packages: \n%s\n\n" "$packages"
    stdout filterdiff --clean --include '*/debian/**' <"$1" || die "Failed to build packaging diff."
}

do_diff_dest() {
    cd "$WORKPARENT"
    our_dsc="$(echo ${SOURCE}_*.dsc)"
    [ -f "$our_dsc" ] || die_gently "Diff failed: No DSC found, was it ever built?"
    raw="$PACKAGING_DIFF.raw"
    touch "$CONTENT_DIFF"
    if [ -n "$DSC_URL" ]; then
        download_from_dsc "$DSC_URL"
        dest_dsc="$(basename "$DSC_URL")"
        debdiff_wrapper "$dest_dsc" "$our_dsc" "$CONTENT_DIFF"
        stdout filterdiff --clean --include '*/debian/**' --exclude '*changelog' <"$CONTENT_DIFF" >"$raw" || die "Failed to discover packaging changes."
    else
        echo '---NEW SOURCE PACKAGE---' >"$raw"
    fi
    if [ -s "$raw" ]; then
        _build_packaging_diff "$CONTENT_DIFF" "$raw" >"$PACKAGING_DIFF"
    fi
}

do_swift_upload() {
    set_swift_creds
    cd "$WORKSPACE"
    retry="Please try regenerating diffs."
    timestamp="$(date --utc '+%Y-%m-%d_%H:%M:%S')"
    [ -n "$SWIFT_TOKEN" ] || die "Failed to determine swift token. $retry"

    diffs=""
    for filename in */*/*.diff; do
        [ -e "$filename" ] || die_gently "No diffs to upload."
        base="$(basename "$filename")"
        confirmed="$(stdout swift upload "$SWIFT_CONTAINER" "$filename" --object-name="$timestamp/$base")" || die "Failed to upload diffs. $retry"
        size="$(wc --lines $filename | awk '{ print $1 }')"
        diffs="$diffs\n$SWIFT_BASE/$SWIFT_CONTAINER/$confirmed ($size lines)"
    done

    set_swift_readable "$SWIFT_CONTAINER" || die "Failed to publish diffs publicly. $retry"
    # shellcheck disable=SC2059
    printf "$diffs"
}

do_delete_swift() {
    set_swift_creds
    loudly swift delete "$SWIFT_CONTAINER" || true
    loudly swift delete "$SWIFT_CONTAINER-excuses" || true
}

do_cached_url_fetcher() {
    dir="/tmp/bileto/cached-urls/$CACHE_BASE"
    mkdir --parents "$dir"
    filename="$(basename "$BRANCH")"
    path="$dir/$filename"
    [ -s "$path" ] || loudly wget --directory-prefix "$dir" --timestamping "$BRANCH" --no-verbose || die "Failed to fetch $BRANCH"
    touch --no-create "$dir" "$path"
    stdout cat "$path"
}

do_find_problematic_diff() {
    cd "$WORKPARENT"
    diff="/tmp/bileto/cached-diffs/${SOURCE}____${TARGET_VERSION}____${DEST_VERSION}.diff"
    if [ ! -f "$diff" ]; then
        loudly mkdir --parents "$(dirname "$diff")"
        download_from_dsc "$DEST_DSC_URL"
        download_from_dsc "$TARGET_DSC_URL"
        debdiff_wrapper "$(basename $TARGET_DSC_URL)" "$(basename $DEST_DSC_URL)" "$diff.raw"
        stdout filterdiff --clean --exclude '*debian/changelog' <"$diff.raw" >"$diff.tmp"
        mv "$diff.tmp" "$diff"
    fi
    loudly touch --no-create "$diff" "$diff.raw"  # delay cron job expiry
    loudly wc --lines "$diff" "$diff.raw"
    if [ -s "$diff" ]; then
        cat "$diff.raw"
    fi
}

do_tag_release() {
    find debian/ -type f -print0 | xargs -0 perl -p -i -e "s/0replaceme/$VERSION/g"
    # Call dch to correctly set the blame / timestamp at the end of the changelog entry
    loudly dch --release --force-distribution --distribution "$SERIES" "" || die "Failed to mark debian/changelog for release."
    # The version won't have been set correctly if the only MP used debcommit though, so fix that
    force_changelog_version
    run_hook pre_release_hook
}

################
# BZR Commands #
################

bzr_do_clean_tags() {
    for tag in $(bzr tags | egrep ' \?$' | awk '{ print $1 }'); do
        loudly bzr tag "$tag" --delete --directory "$BRANCH" || true
    done
}

bzr_do_fill_local_cache() {
    failure="Failed to update local $BRANCH cache."
    if ! persevere loudly bzr pull --directory "$CACHE" --overwrite; then
        loudly bzr branch "$BRANCH" "$CACHE.$$.tmp" || die "$failure"
        loudly rm --recursive --force "$CACHE" || die "$failure"
        loudly mv --verbose "$CACHE.$$.tmp" "$CACHE" || die "$failure"
    fi
    touch --no-create "$CACHE"
}

bzr_do_source_name() {
    bzr_do_fill_local_cache
    parsechangelog "$CACHE" Source
}

bzr_do_source_version() {
    [ -d "$CACHE" ] && where="$CACHE" || where="$WORKDIR"
    parsechangelog "$where" Version
}

bzr_do_branch() {
    loudly bzr init-repo "$WORKPARENT" || die "Failed to initialize bzr repo."
    loudly bzr branch "$CACHE" "$WORKDIR" || die "Failed to branch trunk."
}

bzr_do_merge() {
    authors=""
    output=""
    loudly bzr branch "$BRANCH" "$CACHE_SOURCE" || die "Failed to branch $BRANCH"
    missing="$CACHE_SOURCE.missing"
    stdout bzr missing --theirs "$CACHE_SOURCE" >"$missing"
    for author in $(sed -n 's/^committer: //p' <"$missing" | sort | uniq); do
        output="$author, $output"
        # Author name must be escaped to prevent dangerous eval abuse.
        authors="$authors--author=$(echo "$author" | perl -lpe 's/(\W)/\\$1/gi') "
    done
    loudly bzr merge "$CACHE_SOURCE" || die "Failed to merge $BRANCH"
    if loudly debcommit; then
        echo '--DEBCOMMIT--'
    else
        [ -n "$MESSAGE" ] || die "$NO_COMMIT_MESSAGE"
        msgfile="$WORKPARENT/$$-commit.message"
        echo "$MESSAGE$BUGS$APPROVERS" > "$msgfile"
        eval "bzr commit --unchanged $authors --file=$msgfile" 1>&2 || die "Failed to commit $BRANCH."
        loudly bzr log --limit 1 || die "Failed to display bzr log from $BRANCH."
        echo "$output"
    fi
    rm --recursive --force "$CACHE_SOURCE"
}

bzr_do_tag_release() {
    do_tag_release
    loudly bzr commit --unchanged --message "Releasing $VERSION" || die "Failed to commit release."
    loudly bzr tag --force "$VERSION" || die "Failed to tag release."
    loudly touch "$WORKPARENT/done"
}

bzr_do_expose() {
    loudly bzr push "$BRANCH" --overwrite || die "Failed to push source tree to launchpad."
}

ignore_translators() {
    egrep '^revision-id: ' | egrep --invert-match 'launchpad_translations_on_behalf_of_' | sort
}

bzr_missing_wrapper() {
    retry="bzr: ERROR: Connection closed: Unexpected end of message. Please check connectivity and permissions, and report a bug if problems persist."
    bzr_stderr="$WORKPARENT/$$.bzr.missing.stderr"
    (set -x; bzr missing --show-ids --directory "$@" 2>"$bzr_stderr") || true
    cat "$bzr_stderr" 1>&2
    if grep --ignore-case --quiet "$retry" "$bzr_stderr"; then
        # TODO: Maybe limit the number of retries?
        bzr_missing_wrapper "$@"
    elif egrep --ignore-case --quiet 'error|fail' "$bzr_stderr"; then
        die "bzr missing failed: see log for details."
    fi
}

bzr_do_find_new_commits() {
    # FIXME? https://bugs.launchpad.net/launchpad/+bug/1599631
    me="$(bzr whoami --email)"
    our_cache="$CACHE_DIR/$(escape "$OURS")"
    target_cache="$CACHE_DIR/$(escape "$TARGET")"
    target_has="$WORKPARENT/$$-bzr.target.has"
    missing="$WORKPARENT/$$-bzr.missing"
    # Step 1: Identify commits that the target trunk has that the input branch doesn't (these will be ignored)
    bzr_missing_wrapper "$target_cache" "$BRANCH" --mine-only | tee "$target_has.tmp" 1>&2
    ignore_translators <"$target_has.tmp" >"$target_has"
    # Step 2: Identify new commits that require building.
    bzr_missing_wrapper "$our_cache" "$BRANCH" --theirs-only | tee "$missing" 1>&2
    ignore_translators <"$missing"
    # Step 3: Identify commits that have been deleted from the input branch
    # (commits missing from input branch that aren't already on target trunk, or committed by me)
    bzr_missing_wrapper "$our_cache" "$BRANCH" --mine-only | tee "$missing.tmp" 1>&2
    ignore_translators <"$missing.tmp" | egrep --invert-match "$me" >"$missing"
    comm -23 "$missing" "$target_has"  # Print only lines from $missing not also in $target_has
}

bzr_do_fetch_work_branch() {
    loudly bzr branch "$BRANCH" "$WORKDIR" || die "Failed to branch $BRANCH."
}

bzr_do_merge_trunk() {
    loudly bzr merge "$CACHE" || die "Failed to merge $BRANCH."
    loudly bzr commit -m 'Resync trunk.' || true
}

bzr_do_push_to_trunk() {
    loudly bzr push "$BRANCH" || die "Failed to push to $BRANCH. Check bot team membership and branch ownership."
    # FIXME: Disabled for being **outrageously** slow. This will need to be
    # called from a cron job or something, so it doesn't hold up the finalize.
    #bzr_do_clean_tags
}


################
# GIT Commands #
################

git_fetch_wrapper() {
    persevere loudly git -C "$1" fetch --update-head-ok --force "$2" "$3"
}

git_do_fill_local_cache() {
    failure="Failed to update local $BRANCH cache."
    if ! git_fetch_wrapper "$CACHE" "$BRANCH" "$GIT_REF:$GIT_REF"; then
        tmpdir="$CACHE.$$.tmp"
        mkdir --parents "$tmpdir"
        loudly git init "$tmpdir"
        git_fetch_wrapper "$tmpdir" "$BRANCH" "$GIT_REF:$GIT_REF" || die "$failure"
        loudly rm --recursive --force "$CACHE" || die "$failure"
        loudly mv --verbose "$tmpdir" "$CACHE" || die "$failure"
    fi
    touch --no-create "$CACHE"
}

git_do_source_name() {
    git_do_fill_local_cache
    stdout git -C "$CACHE" show "$GIT_REF:debian/changelog" | awk 'NR==1{ print $1 }'
}

git_do_source_version() {
    if [ "$BRANCH" = 'WORKDIR' ]; then
        parsechangelog "$WORKDIR" Version
    else
        stdout git -C "$CACHE" show "$GIT_REF:debian/changelog" | awk -vFS='(' -vRS=')' 'NR==1{ print $2 }'
    fi
}

git_do_branch() {
    loudly mkdir --parents "$WORKDIR" || die "Failed to create $WORKDIR."
    loudly git init "$WORKDIR" || die "Failed to init git repo in $WORKDIR."
    git_fetch_wrapper "$WORKDIR" "$CACHE" "$GIT_TARGET_REF:$GIT_REF" || die "Failed to fetch $BRANCH:$GIT_TARGET_REF."
    loudly git -C "$WORKDIR" checkout --force "$UNIQUE_ID" || die "Failed to checkout $UNIQUE_ID."
}

git_do_merge() {
    msgfile="$WORKPARENT/$$-commit.message"
    git_fetch_wrapper "$WORKDIR" "$BRANCH" "$GIT_SOURCE_REF:refs/heads/bileto-merge-source" || die "Failed to fetch $BRANCH."
    authors="$(stdout git log HEAD..bileto-merge-source --format='format:%cn <%ce>' | sort | uniq | perl -p -e 's/\n/, /')"
    loudly git merge --no-ff --no-commit bileto-merge-source || die "Failed to merge $BRANCH."
    # Extract commit message from supplied debian/changelog, if any
    stdout git diff --cached | filterdiff --clean --include '*/debian/changelog' | sed --quiet --expression 's/^+  //p' > "$msgfile"
    if [ -s "$msgfile" ]; then
        # Alert the calling process not to attempt to generate a changelog with dch
        echo '--DEBCOMMIT--'
    else
        [ -n "$MESSAGE" ] || die "$NO_COMMIT_MESSAGE"
        echo "$MESSAGE$BUGS$APPROVERS" > "$msgfile"
        echo "$authors"
    fi
    loudly git commit --allow-empty --all --file=$msgfile --author="$authors" || die "Failed to commit $BRANCH."
}

git_do_tag_release() {
    do_tag_release
    tag="$(echo $VERSION | tr '~:' '_%')"
    loudly git commit --all --message "Releasing $VERSION" || die "Failed to commit release."
    loudly git tag --force --local-user "$(gpg_key)" "$tag" --message "Releasing $VERSION" || die "Failed to tag release."
    loudly touch "$WORKPARENT/done"
}

git_do_expose() {
    loudly git push "$BRANCH" "$GIT_REF:$GIT_REF" --prune --force --tags || die "Failed to push source tree to $BRANCH:$GIT_REF."
}

git_do_find_new_commits() {
    me="$(git config user.email)"
    format="--format=format:revision-id: %ce %ci (%cr) %H"
    target_has="$WORKPARENT/$$-git.target.has"
    missing="$WORKPARENT/$$-git.missing"
    cd "$CACHE"
    git_fetch_wrapper "$CACHE" "$TARGET" "$GIT_TARGET_REF:$GIT_WORKING_REF-merge-target" || die "Failed to fetch $TARGET."
    git_fetch_wrapper "$CACHE" "$BRANCH" "$GIT_SOURCE_REF:$GIT_WORKING_REF-merge-source" || die "Failed to fetch $BRANCH."
    # Step 1: Identify commits that the target trunk has that the input branch doesn't (these will be ignored)
    stdout git log "$GIT_WORKING_REF-merge-source..$GIT_WORKING_REF-merge-target" "$format" | tee "$target_has.tmp" 1>&2
    ignore_translators <"$target_has.tmp" >"$target_has"
    # Step 2: Identify new commits that require building.
    stdout git log "$GIT_WORKING_REF..$GIT_WORKING_REF-merge-source" "$format" | tee "$missing" 1>&2
    ignore_translators <"$missing"
    # Step 3: Identify commits that have been deleted from the input branch
    # (commits missing from input branch that aren't already on target trunk, or committed by me)
    stdout git log "$GIT_WORKING_REF-merge-source..$GIT_WORKING_REF" --first-parent "$format" | tee "$missing.tmp" 1>&2
    ignore_translators <"$missing.tmp" | egrep --invert-match "$me" >"$missing"
    comm -23 "$missing" "$target_has"  # Print only lines from $missing not also in $target_has
}

git_do_fetch_work_branch() {
    GIT_TARGET_REF="$GIT_WORKING_REF"
    git_do_branch
}

git_do_merge_trunk() {
    git_fetch_wrapper "$WORKDIR" "$CACHE" "$GIT_TARGET_REF:refs/heads/bileto-target-master" || die "Failed to fetch $BRANCH."
    loudly git merge bileto-target-master || die "Failed to merge $BRANCH:$GIT_TARGET_REF into $GIT_REF."
}

git_do_push_to_trunk() {
    loudly git push "$BRANCH" "$GIT_REF:$GIT_TARGET_REF" --force --tags || die "Failed to push $GIT_REF to $BRANCH:$GIT_TARGET_REF. Check bot team membership and branch ownership."
}


ACTION="do_$ACTION"
VCS_ACTION="${VCS}_$ACTION"

# shellcheck disable=SC2039
type "$VCS_ACTION" 2>/dev/null 1>&2 && $VCS_ACTION "$@" || $ACTION "$@"