Merge branch 'js/ci-github-workflow-markup'

Update the GitHub workflow support to make it quicker to get to the
failing test.

* js/ci-github-workflow-markup:
  ci: call `finalize_test_case_output` a little later
  ci(github): mention where the full logs can be found
  ci: use `--github-workflow-markup` in the GitHub workflow
  ci(github): avoid printing test case preamble twice
  ci(github): skip the logs of the successful test cases
  ci: optionally mark up output in the GitHub workflow
  ci/run-build-and-tests: add some structure to the GitHub workflow output
  ci: make it easier to find failed tests' logs in the GitHub workflow
  ci/run-build-and-tests: take a more high-level view
  test(junit): avoid line feeds in XML attributes
  tests: refactor --write-junit-xml code
  ci: fix code style
pull/1275/head
Junio C Hamano 4 months ago
commit fc5a070f59
  1. 12
      .github/workflows/main.yml
  2. 83
      ci/lib.sh
  3. 14
      ci/run-build-and-tests.sh
  4. 5
      ci/run-test-slice.sh
  5. 6
      t/test-lib-functions.sh
  6. 56
      t/test-lib-github-workflow-markup.sh
  7. 132
      t/test-lib-junit.sh
  8. 128
      t/test-lib.sh

@ -119,10 +119,6 @@ jobs:
- name: test
shell: bash
run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10
- name: ci/print-test-failures.sh
if: failure()
shell: bash
run: ci/print-test-failures.sh
- name: Upload failed tests' directories
if: failure() && env.FAILED_TEST_ARTIFACTS != ''
uses: actions/upload-artifact@v2
@ -204,10 +200,6 @@ jobs:
env:
NO_SVN_TESTS: 1
run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10
- name: ci/print-test-failures.sh
if: failure()
shell: bash
run: ci/print-test-failures.sh
- name: Upload failed tests' directories
if: failure() && env.FAILED_TEST_ARTIFACTS != ''
uses: actions/upload-artifact@v2
@ -261,8 +253,6 @@ jobs:
- uses: actions/checkout@v2
- run: ci/install-dependencies.sh
- run: ci/run-build-and-tests.sh
- run: ci/print-test-failures.sh
if: failure()
- name: Upload failed tests' directories
if: failure() && env.FAILED_TEST_ARTIFACTS != ''
uses: actions/upload-artifact@v2
@ -292,8 +282,6 @@ jobs:
- uses: actions/checkout@v1
- run: ci/install-docker-dependencies.sh
- run: ci/run-build-and-tests.sh
- run: ci/print-test-failures.sh
if: failure()
- name: Upload failed tests' directories
if: failure() && env.FAILED_TEST_ARTIFACTS != ''
uses: actions/upload-artifact@v1

@ -1,5 +1,50 @@
# Library of functions shared by all CI scripts
if test true != "$GITHUB_ACTIONS"
then
begin_group () { :; }
end_group () { :; }
group () {
shift
"$@"
}
set -x
else
begin_group () {
need_to_end_group=t
echo "::group::$1" >&2
set -x
}
end_group () {
test -n "$need_to_end_group" || return 0
set +x
need_to_end_group=
echo '::endgroup::' >&2
}
trap end_group EXIT
group () {
set +x
begin_group "$1"
shift
"$@"
res=$?
end_group
return $res
}
begin_group "CI setup"
fi
# Set 'exit on error' for all CI scripts to let the caller know that
# something went wrong.
#
# We already enabled tracing executed commands earlier. This helps by showing
# how # environment variables are set and and dependencies are installed.
set -e
skip_branch_tip_with_tag () {
# Sometimes, a branch is pushed at the same time the tag that points
# at the same commit as the tip of the branch is pushed, and building
@ -69,8 +114,7 @@ skip_good_tree () {
exit 0
}
check_unignored_build_artifacts ()
{
check_unignored_build_artifacts () {
! git ls-files --other --exclude-standard --error-unmatch \
-- ':/*' 2>/dev/null ||
{
@ -79,18 +123,16 @@ check_unignored_build_artifacts ()
}
}
handle_failed_tests () {
return 1
}
# GitHub Action doesn't set TERM, which is required by tput
export TERM=${TERM:-dumb}
# Clear MAKEFLAGS that may come from the outside world.
export MAKEFLAGS=
# Set 'exit on error' for all CI scripts to let the caller know that
# something went wrong.
# Set tracing executed commands, primarily setting environment variables
# and installing dependencies.
set -ex
if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI"
then
CI_TYPE=azure-pipelines
@ -124,11 +166,31 @@ then
CI_JOB_ID="$GITHUB_RUN_ID"
CC="${CC_PACKAGE:-${CC:-gcc}}"
DONT_SKIP_TAGS=t
handle_failed_tests () {
mkdir -p t/failed-test-artifacts
echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV
for test_exit in t/test-results/*.exit
do
test 0 != "$(cat "$test_exit")" || continue
test_name="${test_exit%.exit}"
test_name="${test_name##*/}"
printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n"
echo "The full logs are in the artifacts attached to this run."
cat "t/test-results/$test_name.markup"
trash_dir="t/trash directory.$test_name"
cp "t/test-results/$test_name.out" t/failed-test-artifacts/
tar czf t/failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir"
done
return 1
}
cache_dir="$HOME/none"
export GIT_PROVE_OPTS="--timer --jobs 10"
export GIT_TEST_OPTS="--verbose-log -x"
export GIT_TEST_OPTS="--verbose-log -x --github-workflow-markup"
MAKEFLAGS="$MAKEFLAGS --jobs=10"
test windows != "$CI_OS_NAME" ||
GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
@ -211,3 +273,6 @@ linux-leaks)
esac
MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
end_group
set -x

@ -10,7 +10,7 @@ windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";;
*) ln -s "$cache_dir/.prove" t/.prove;;
esac
export MAKE_TARGETS="all test"
run_tests=t
case "$jobname" in
linux-gcc)
@ -41,14 +41,16 @@ pedantic)
# Don't run the tests; we only care about whether Git can be
# built.
export DEVOPTS=pedantic
export MAKE_TARGETS=all
run_tests=
;;
esac
# Any new "test" targets should not go after this "make", but should
# adjust $MAKE_TARGETS. Otherwise compilation-only targets above will
# start running tests.
make $MAKE_TARGETS
group Build make
if test -n "$run_tests"
then
group "Run tests" make test ||
handle_failed_tests
fi
check_unignored_build_artifacts
save_good_tree

@ -10,8 +10,9 @@ windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";;
*) ln -s "$cache_dir/.prove" t/.prove;;
esac
make --quiet -C t T="$(cd t &&
group "Run tests" make --quiet -C t T="$(cd t &&
./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh |
tr '\n' ' ')"
tr '\n' ' ')" ||
handle_failed_tests
check_unignored_build_artifacts

@ -795,7 +795,7 @@ test_verify_prereq () {
}
test_expect_failure () {
test_start_
test_start_ "$@"
test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
test "$#" = 2 ||
BUG "not 2 or 3 parameters to test-expect-failure"
@ -803,6 +803,7 @@ test_expect_failure () {
export test_prereq
if ! test_skip "$@"
then
test -n "$test_skip_test_preamble" ||
say >&3 "checking known breakage of $TEST_NUMBER.$test_count '$1': $2"
if test_run_ "$2" expecting_failure
then
@ -815,7 +816,7 @@ test_expect_failure () {
}
test_expect_success () {
test_start_
test_start_ "$@"
test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
test "$#" = 2 ||
BUG "not 2 or 3 parameters to test-expect-success"
@ -823,6 +824,7 @@ test_expect_success () {
export test_prereq
if ! test_skip "$@"
then
test -n "$test_skip_test_preamble" ||
say >&3 "expecting success of $TEST_NUMBER.$test_count '$1': $2"
if test_run_ "$2"
then

@ -0,0 +1,56 @@
# Library of functions to mark up test scripts' output suitable for
# pretty-printing it in GitHub workflows.
#
# Copyright (c) 2022 Johannes Schindelin
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see http://www.gnu.org/licenses/ .
#
# The idea is for `test-lib.sh` to source this file when run in GitHub
# workflows; these functions will then override (empty) functions
# that are are called at the appropriate times during the test runs.
test_skip_test_preamble=t
start_test_output () {
test -n "$GIT_TEST_TEE_OUTPUT_FILE" ||
die "--github-workflow-markup requires --verbose-log"
github_markup_output="${GIT_TEST_TEE_OUTPUT_FILE%.out}.markup"
>$github_markup_output
GIT_TEST_TEE_OFFSET=0
}
# No need to override start_test_case_output
finalize_test_case_output () {
test_case_result=$1
shift
case "$test_case_result" in
failure)
echo >>$github_markup_output "::error::failed: $this_test.$test_count $1"
;;
fixed)
echo >>$github_markup_output "::notice::fixed: $this_test.$test_count $1"
;;
ok)
# Exit without printing the "ok" tests
return
;;
esac
echo >>$github_markup_output "::group::$test_case_result: $this_test.$test_count $*"
test-tool >>$github_markup_output path-utils skip-n-bytes \
"$GIT_TEST_TEE_OUTPUT_FILE" $GIT_TEST_TEE_OFFSET
echo >>$github_markup_output "::endgroup::"
}
# No need to override finalize_test_output

@ -0,0 +1,132 @@
# Library of functions to format test scripts' output in JUnit XML
# format, to support Git's test suite result to be presented in an
# easily digestible way on Azure Pipelines.
#
# Copyright (c) 2022 Johannes Schindelin
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see http://www.gnu.org/licenses/ .
#
# The idea is for `test-lib.sh` to source this file when the user asks
# for JUnit XML; these functions will then override (empty) functions
# that are are called at the appropriate times during the test runs.
start_test_output () {
junit_xml_dir="$TEST_OUTPUT_DIRECTORY/out"
mkdir -p "$junit_xml_dir"
junit_xml_base=${1##*/}
junit_xml_path="$junit_xml_dir/TEST-${junit_xml_base%.sh}.xml"
junit_attrs="name=\"${junit_xml_base%.sh}\""
junit_attrs="$junit_attrs timestamp=\"$(TZ=UTC \
date +%Y-%m-%dT%H:%M:%S)\""
write_junit_xml --truncate "<testsuites>" " <testsuite $junit_attrs>"
junit_suite_start=$(test-tool date getnanos)
if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
then
GIT_TEST_TEE_OFFSET=0
fi
}
start_test_case_output () {
junit_start=$(test-tool date getnanos)
}
finalize_test_case_output () {
test_case_result=$1
shift
case "$test_case_result" in
ok)
set "$*"
;;
failure)
junit_insert="<failure message=\"not ok $test_count -"
junit_insert="$junit_insert $(xml_attr_encode --no-lf "$1")\">"
junit_insert="$junit_insert $(xml_attr_encode \
"$(if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
then
test-tool path-utils skip-n-bytes \
"$GIT_TEST_TEE_OUTPUT_FILE" $GIT_TEST_TEE_OFFSET
else
printf '%s\n' "$@" | sed 1d
fi)")"
junit_insert="$junit_insert</failure>"
if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
then
junit_insert="$junit_insert<system-err>$(xml_attr_encode \
"$(cat "$GIT_TEST_TEE_OUTPUT_FILE")")</system-err>"
fi
set "$1" " $junit_insert"
;;
fixed)
set "$* (breakage fixed)"
;;
broken)
set "$* (known breakage)"
;;
skip)
message="$(xml_attr_encode --no-lf "$skipped_reason")"
set "$1" " <skipped message=\"$message\" />"
;;
esac
junit_attrs="name=\"$(xml_attr_encode --no-lf "$this_test.$test_count $1")\""
shift
junit_attrs="$junit_attrs classname=\"$this_test\""
junit_attrs="$junit_attrs time=\"$(test-tool \
date getnanos $junit_start)\""
write_junit_xml "$(printf '%s\n' \
" <testcase $junit_attrs>" "$@" " </testcase>")"
junit_have_testcase=t
}
finalize_test_output () {
if test -n "$junit_xml_path"
then
test -n "$junit_have_testcase" || {
junit_start=$(test-tool date getnanos)
write_junit_xml_testcase "all tests skipped"
}
# adjust the overall time
junit_time=$(test-tool date getnanos $junit_suite_start)
sed -e "s/\(<testsuite.*\) time=\"[^\"]*\"/\1/" \
-e "s/<testsuite [^>]*/& time=\"$junit_time\"/" \
-e '/^ *<\/testsuite/d' \
<"$junit_xml_path" >"$junit_xml_path.new"
mv "$junit_xml_path.new" "$junit_xml_path"
write_junit_xml " </testsuite>" "</testsuites>"
write_junit_xml=
fi
}
write_junit_xml () {
case "$1" in
--truncate)
>"$junit_xml_path"
junit_have_testcase=
shift
;;
esac
printf '%s\n' "$@" >>"$junit_xml_path"
}
xml_attr_encode () {
if test "x$1" = "x--no-lf"
then
shift
printf '%s' "$*" | test-tool xml-encode
else
printf '%s\n' "$@" | test-tool xml-encode
fi
}

@ -137,6 +137,12 @@ mark_option_requires_arg () {
store_arg_to=$2
}
# These functions can be overridden e.g. to output JUnit XML
start_test_output () { :; }
start_test_case_output () { :; }
finalize_test_case_output () { :; }
finalize_test_output () { :; }
parse_option () {
local opt="$1"
@ -196,7 +202,10 @@ parse_option () {
tee=t
;;
--write-junit-xml)
write_junit_xml=t
. "$TEST_DIRECTORY/test-lib-junit.sh"
;;
--github-workflow-markup)
. "$TEST_DIRECTORY/test-lib-github-workflow-markup.sh"
;;
--stress)
stress=t ;;
@ -664,7 +673,7 @@ exec 6<&0
exec 7>&2
_error_exit () {
finalize_junit_xml
finalize_test_output
GIT_EXIT_OK=t
exit 1
}
@ -774,35 +783,13 @@ trap '{ code=$?; set +x; } 2>/dev/null; exit $code' INT TERM HUP
# the test_expect_* functions instead.
test_ok_ () {
if test -n "$write_junit_xml"
then
write_junit_xml_testcase "$*"
fi
test_success=$(($test_success + 1))
say_color "" "ok $test_count - $@"
finalize_test_case_output ok "$@"
}
test_failure_ () {
if test -n "$write_junit_xml"
then
junit_insert="<failure message=\"not ok $test_count -"
junit_insert="$junit_insert $(xml_attr_encode "$1")\">"
junit_insert="$junit_insert $(xml_attr_encode \
"$(if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
then
test-tool path-utils skip-n-bytes \
"$GIT_TEST_TEE_OUTPUT_FILE" $GIT_TEST_TEE_OFFSET
else
printf '%s\n' "$@" | sed 1d
fi)")"
junit_insert="$junit_insert</failure>"
if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
then
junit_insert="$junit_insert<system-err>$(xml_attr_encode \
"$(cat "$GIT_TEST_TEE_OUTPUT_FILE")")</system-err>"
fi
write_junit_xml_testcase "$1" " $junit_insert"
fi
failure_label=$1
test_failure=$(($test_failure + 1))
say_color error "not ok $test_count - $1"
shift
@ -812,24 +799,19 @@ test_failure_ () {
say_color error "1..$test_count"
_error_exit
fi
finalize_test_case_output failure "$failure_label" "$@"
}
test_known_broken_ok_ () {
if test -n "$write_junit_xml"
then
write_junit_xml_testcase "$* (breakage fixed)"
fi
test_fixed=$(($test_fixed+1))
say_color error "ok $test_count - $@ # TODO known breakage vanished"
finalize_test_case_output fixed "$@"
}
test_known_broken_failure_ () {
if test -n "$write_junit_xml"
then
write_junit_xml_testcase "$* (known breakage)"
fi
test_broken=$(($test_broken+1))
say_color warn "not ok $test_count - $@ # TODO known breakage"
finalize_test_case_output broken "$@"
}
test_debug () {
@ -1104,10 +1086,7 @@ test_start_ () {
test_count=$(($test_count+1))
maybe_setup_verbose
maybe_setup_valgrind
if test -n "$write_junit_xml"
then
junit_start=$(test-tool date getnanos)
fi
start_test_case_output "$@"
}
test_finish_ () {
@ -1158,15 +1137,10 @@ test_skip () {
case "$to_skip" in
t)
if test -n "$write_junit_xml"
then
message="$(xml_attr_encode "$skipped_reason")"
write_junit_xml_testcase "$1" \
" <skipped message=\"$message\" />"
fi
say_color skip "ok $test_count # skip $1 ($skipped_reason)"
: true
finalize_test_case_output skip "$@"
;;
*)
false
@ -1179,53 +1153,6 @@ test_at_end_hook_ () {
:
}
write_junit_xml () {
case "$1" in
--truncate)
>"$junit_xml_path"
junit_have_testcase=
shift
;;
esac
printf '%s\n' "$@" >>"$junit_xml_path"
}
xml_attr_encode () {
printf '%s\n' "$@" | test-tool xml-encode
}
write_junit_xml_testcase () {
junit_attrs="name=\"$(xml_attr_encode "$this_test.$test_count $1")\""
shift
junit_attrs="$junit_attrs classname=\"$this_test\""
junit_attrs="$junit_attrs time=\"$(test-tool \
date getnanos $junit_start)\""
write_junit_xml "$(printf '%s\n' \
" <testcase $junit_attrs>" "$@" " </testcase>")"
junit_have_testcase=t
}
finalize_junit_xml () {
if test -n "$write_junit_xml" && test -n "$junit_xml_path"
then
test -n "$junit_have_testcase" || {
junit_start=$(test-tool date getnanos)
write_junit_xml_testcase "all tests skipped"
}
# adjust the overall time
junit_time=$(test-tool date getnanos $junit_suite_start)
sed -e "s/\(<testsuite.*\) time=\"[^\"]*\"/\1/" \
-e "s/<testsuite [^>]*/& time=\"$junit_time\"/" \
-e '/^ *<\/testsuite/d' \
<"$junit_xml_path" >"$junit_xml_path.new"
mv "$junit_xml_path.new" "$junit_xml_path"
write_junit_xml " </testsuite>" "</testsuites>"
write_junit_xml=
fi
}
test_atexit_cleanup=:
test_atexit_handler () {
# In a succeeding test script 'test_atexit_handler' is invoked
@ -1248,7 +1175,7 @@ test_done () {
# removed, so the commands can access pidfiles and socket files.
test_atexit_handler
finalize_junit_xml
finalize_test_output
if test -z "$HARNESS_ACTIVE"
then
@ -1539,22 +1466,7 @@ fi
# in subprocesses like git equals our $PWD (for pathname comparisons).
cd -P "$TRASH_DIRECTORY" || exit 1
if test -n "$write_junit_xml"
then
junit_xml_dir="$TEST_OUTPUT_DIRECTORY/out"
mkdir -p "$junit_xml_dir"
junit_xml_base=${0##*/}
junit_xml_path="$junit_xml_dir/TEST-${junit_xml_base%.sh}.xml"
junit_attrs="name=\"${junit_xml_base%.sh}\""
junit_attrs="$junit_attrs timestamp=\"$(TZ=UTC \
date +%Y-%m-%dT%H:%M:%S)\""
write_junit_xml --truncate "<testsuites>" " <testsuite $junit_attrs>"
junit_suite_start=$(test-tool date getnanos)
if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
then
GIT_TEST_TEE_OFFSET=0
fi
fi
start_test_output "$0"
# Convenience
# A regexp to match 5 and 35 hexdigits

Loading…
Cancel
Save