tests.sh

#!/usr/bin/env bash

set -euo pipefail

if [[ ! -e flake.nix ]]; then
    echo "missing flake.nix !!!" >&2
fi

NIXPKGS_UPSTREAM="https://github.com/NixOS/nixpkgs"

AUTO_BISECT_GOOD=""  # good commit for git bisect on failure
GREP_FILTER=""
RANDOM_ORDER=""  # useful for quicker mass-bug-detecting on release upgrades
PRINT_OUT=""
while [[ $# -gt 0 ]]; do
    case "$1" in
        --auto-bisect)
            AUTO_BISECT_GOOD="$2"
            shift 2
            ;;
        --grep)
            GREP_FILTER="$2"
            shift 2
            ;;
        --print-out)
            PRINT_OUT="y"
            shift 1
            ;;
        --random-order)
            RANDOM_ORDER="y"
            shift 1
            ;;
        *)
            echo "unknown argument: $1" >&2
            exit 2
            ;;
    esac
done

filter() {
    if [[ -n "$GREP_FILTER" ]]; then
        grep "$GREP_FILTER"
    elif [[ -n "$RANDOM_ORDER" ]]; then
        shuf
    else
        cat
    fi
}

architecture="$( nix eval --impure --expr "builtins.currentSystem" )"
targetAttr=".#x-banananetwork_ci-targets.${architecture}"

# targets which are to be checked
targets=()

mapfile -t test_targets < <( nix eval --raw "${targetAttr}.testTargetsText" | filter )
targets+=("${test_targets[@]}")

mapfile -t build_targets < <( nix eval --raw "${targetAttr}.buildTargetsText" | filter )
targets+=("${build_targets[@]}")

if [[ -n "$PRINT_OUT" ]]; then
    echo "would build following targets:"
    for target in "${targets[@]}"; do
        echo "  - $target"
    done
    exit 0
fi

# - checks.* often have empty outputs or at least small ones
#   - checks are e.g. NixOS tests
# -> holding a GC root for them indefinitely is cheap
# -> prevents rebuilding the same check target in the future
#   - e.g. on a git bisect, where historic commits are rechecked
# - other builds are separated s.t. only their latest build is kept, to save space
#   - other builds are e.g. devShells.*, nixosConfigurations.*, packages.*, …
is_test_target() {
    case "$1" in
        checks.*) return 0 ;;
        *) return 1 ;;
    esac
}

rememberGC() {
    if [[ ${CI_GCROOT:-} == "" ]]; then
        return 0
    fi
    local target="$1"
    local commit
    if ! commit="$(git rev-parse --short HEAD 2>/dev/null)"; then
        echo "rememberGC: cannot determine commit hash (not in a git repo?)" >&2
        return 1
    fi
    local new_loc
    if is_test_target "$target"; then
        new_loc="$CI_GCROOT/checks/$commit/$target"
    else
        new_loc="$CI_GCROOT/builds/$commit/$target"
    fi
    mkdir -p "$(dirname "$new_loc")"
    mv ./result "$new_loc"
    nix-store --add-root "$new_loc" --realise "$new_loc"
}

# git bisect the failure of a single target to its first failing commit
bisect_failure() {
    local target="$1"
    # avoid duplicated error logging
    # last lines should be "For full logs, run: nix log …"
    local test_runner='
        set -euo pipefail
        ./build_remote.sh .#'"${target@Q}"' 2>&1 | \
        tail --lines=2
    '
    echo "attempt bisecting error" >&2
    git bisect reset >/dev/null
    git bisect start >/dev/null
    git bisect bad HEAD >/dev/null
    git bisect good "$AUTO_BISECT_GOOD" >/dev/null
    git bisect run bash -c "$test_runner"
    local bad_commit="$(git rev-parse --short refs/bisect/bad)"
    if git show --format=%B --no-patch "$bad_commit" \
        | grep -q "Updated input 'nixpkgs':"; then
        # commit generated by `nix flake update --commit-lock-file`, e.g.:
        #   • Updated input 'nixpkgs':
        #       'github:nixos/nixpkgs/<from>?narHash=…' (old date)
        #     → 'github:nixos/nixpkgs/<to>?narHash=…'   (new date)
        local commit_body nixpkgs_hashes fromCommit toCommit
        commit_body="$(git show --format=%B --no-patch "$bad_commit")"
        nixpkgs_hashes="$(awk '
            /Updated input .nixpkgs.:/ { on = 1 }
            on && /^[[:space:]]*$/ { exit }
            on { print }
        ' <<<"$commit_body" | grep -oE '[0-9a-f]{40}')"
        fromCommit="$(sed -n '1p' <<<"$nixpkgs_hashes")"
        toCommit="$(sed -n '2p' <<<"$nixpkgs_hashes")"
        nixpkgs_bisect_failure "$fromCommit" "$toCommit" "$target"
    fi
    git bisect reset >/dev/null
}

nixpkgs_bisect_failure() {
    local fromCommit="$1"
    local toCommit="$2"
    local target="$3"
    local nixpkgs_repo="$(mktemp -d nixpkgs-bisect.tmp.XXXXXXXXXX)"
    trap 'rm -rf '"${nixpkgs_repo@Q}" RETURN
    local test_runner='
        set -euo pipefail
        cd '"$(pwd)"' && \
        ./build_remote.sh --override-input nixpkgs '"${nixpkgs_repo@Q}"' .#'"${target@Q}"' 2>&1 | \
        tail --lines=2
    '
    echo "continue bisecting failure in nixpkgs" >&2
    git clone "$NIXPKGS_UPSTREAM" "$nixpkgs_repo" >/dev/null
    pushd "$nixpkgs_repo"
    git bisect reset >/dev/null
    git bisect start >/dev/null
    git bisect bad "$toCommit" >/dev/null
    git bisect good "$fromCommit" >/dev/null
    git bisect run bash -c "$test_runner"
    # Perform bisect on the nixpkgs repository
    git bisect reset >/dev/null
    popd
}

set -x

for target in "${targets[@]}"; do
    for i in 0 1 last; do
        if ./build_remote.sh "$@" .#"$target"; then
            rememberGC "$target"
            break  # continue outside
        elif [[ $i == "last" ]]; then
            echo "last attempt failed, forward error" >&2
            if [[ -n "$AUTO_BISECT_GOOD" ]]; then
                bisect_failure "$target"
            fi
            exit 1
        else
            echo "attempt no. $i failed, retry" >&2
        fi
    done
done