From 516f806d58fedda6d64984dfd02c4a3eb12caef3 Mon Sep 17 00:00:00 2001
From: Ingo Karkat <swdev@ingo-karkat.de>
Date: Wed, 25 Jan 2012 15:10:42 +0100
Subject: [PATCH 1/4] test-lib: Always print output differences, not just in
 verbose mode.

Differences in the expected and actual todo.sh output are critical to analyzing the error, so they should always be printed, not just when the -verbose argument is given.

This refactoring moves the test_cmp call from the command under test (as seen from test-lib's perspective) into a new assertion test_expect_output, derived from test_expect_success.
---
 tests/test-lib.sh | 34 +++++++++++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/tests/test-lib.sh b/tests/test-lib.sh
index 29f3246..e69f484 100644
--- a/tests/test-lib.sh
+++ b/tests/test-lib.sh
@@ -180,7 +180,7 @@ test_failure_ () {
 	test_failure=$(($test_failure + 1))
 	say_color error "FAIL $test_count: $1"
 	shift
-	echo "$@" | sed -e 's/^/	/'
+	echo "$@"
 	test "$immediate" = "" || { trap - EXIT; exit 1; }
 }
 
@@ -260,6 +260,30 @@ test_expect_success () {
 	echo >&3 ""
 }
 
+test_expect_output () {
+	test "$#" = 2 ||
+	error "bug in the test script: not 2 parameters to test-expect-output"
+	if ! test_skip "$@"
+	then
+		say >&3 "expecting success and output: $2"
+		test_run_ "$2"
+		if [ "$?" = 0 -a "$eval_ret" = 0 ]
+		then
+			cmp_output=$(test_cmp expect output)
+			if [ "$?" = 0 ]
+			then
+				test_ok_ "$1"
+			else
+				test_failure_ "$@" "
+$cmp_output"
+			fi
+		else
+			test_failure_ "$@"
+		fi
+	fi
+	echo >&3 ""
+}
+
 test_expect_code () {
 	test "$#" = 3 ||
 	error "bug in the test script: not 3 parameters to test-expect-code"
@@ -542,9 +566,9 @@ test_todo_session () {
 	"")
 	    if [ ! -z "$cmd" ]; then
 		if [ $status = 0 ]; then
-		    test_expect_success "$1 $subnum" "$cmd > output && test_cmp expect output"
+		    test_expect_output "$1 $subnum" "$cmd > output"
 		else
-		    test_expect_success "$1 $subnum" "$cmd > output ; test \$? = $status && test_cmp expect output"
+		    test_expect_output "$1 $subnum" "$cmd > output ; test \$? = $status"
 		fi
 
 		subnum=$(($subnum + 1))
@@ -560,9 +584,9 @@ test_todo_session () {
     done
     if [ ! -z "$cmd" ]; then
 	if [ $status = 0 ]; then
-	    test_expect_success "$1 $subnum" "$cmd > output && test_cmp expect output"
+	    test_expect_output "$1 $subnum" "$cmd > output"
 	else
-	    test_expect_success "$1 $subnum" "$cmd > output ; test \$? = $status && test_cmp expect output"
+	    test_expect_output "$1 $subnum" "$cmd > output ; test \$? = $status"
 	fi
     fi
 }

From 189779c6de52efca7f783478fb25022bcb7be776 Mon Sep 17 00:00:00 2001
From: Ingo Karkat <swdev@ingo-karkat.de>
Date: Wed, 25 Jan 2012 15:23:57 +0100
Subject: [PATCH 2/4] test-lib: Separate function for exit code assertion.

Remove the check for the todo.sh exit code (scripted via "=== N") from the command under test into a separate assertion test_expect_code_and_output. This allows for reporting of expected vs. actual exit code (also in the default non-verbose mode), and unexpected output from the same test is now reported, too.
---
 tests/test-lib.sh | 34 +++++++++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/tests/test-lib.sh b/tests/test-lib.sh
index e69f484..2368bd3 100644
--- a/tests/test-lib.sh
+++ b/tests/test-lib.sh
@@ -284,6 +284,33 @@ $cmp_output"
 	echo >&3 ""
 }
 
+test_expect_code_and_output () {
+	test "$#" = 3 ||
+	error "bug in the test script: not 3 parameters to test-expect-code-and-output"
+	if ! test_skip "$@"
+	then
+		say >&3 "expecting exit code $1 and output: $3"
+		test_run_ "$3"
+		if [ "$?" = 0 -a "$eval_ret" = "$1" ]
+		then
+			cmp_output=$(test_cmp expect output)
+			if [ "$?" = 0 ]
+			then
+				test_ok_ "$2"
+			else
+				test_failure_ "$2" "$3" "
+$cmp_output"
+			fi
+		else
+			cmp_output=$(test_cmp expect output)
+			test_failure_ "$2" "$3" "
+* expected exit code $1, actual ${eval_ret}${cmp_output:+
+}${cmp_output}"
+		fi
+	fi
+	echo >&3 ""
+}
+
 test_expect_code () {
 	test "$#" = 3 ||
 	error "bug in the test script: not 3 parameters to test-expect-code"
@@ -295,7 +322,8 @@ test_expect_code () {
 		then
 			test_ok_ "$2"
 		else
-			test_failure_ "$@"
+			test_failure_ "$2" "$3" "
+* expected exit code $1, actual ${eval_ret}"
 		fi
 	fi
 	echo >&3 ""
@@ -568,7 +596,7 @@ test_todo_session () {
 		if [ $status = 0 ]; then
 		    test_expect_output "$1 $subnum" "$cmd > output"
 		else
-		    test_expect_output "$1 $subnum" "$cmd > output ; test \$? = $status"
+		    test_expect_code_and_output "$status" "$1 $subnum" "$cmd > output"
 		fi
 
 		subnum=$(($subnum + 1))
@@ -586,7 +614,7 @@ test_todo_session () {
 	if [ $status = 0 ]; then
 	    test_expect_output "$1 $subnum" "$cmd > output"
 	else
-	    test_expect_output "$1 $subnum" "$cmd > output ; test \$? = $status"
+	    test_expect_code_and_output "$status" "$1 $subnum" "$cmd > output"
 	fi
     fi
 }

From ebe9fb868b9d07ddcb5823f5d6beb4d99d93bfb1 Mon Sep 17 00:00:00 2001
From: Ingo Karkat <swdev@ingo-karkat.de>
Date: Wed, 25 Jan 2012 15:45:35 +0100
Subject: [PATCH 3/4] test-lib: Show full todo.sh output in verbose mode.

Now that differences in the output (and exit code) are already printed by default, we can make the verbose mode actually "verbose" by including all todo.sh output generated during the test run. This may help in reviewing the tests and for troubleshooting.
By moving the redirection to the output file to test_run_(), all testing-related embellishments have been removed from the command under test itself, resulting in much cleaner test messages.
Additionally, also capture stderr in output. todo.sh itself currently prints everything to stdout (but the die() output probably belongs to stderr), so as of now, that has no consequences, but seems to be more consistent and future-proof.
---
 tests/test-lib.sh | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tests/test-lib.sh b/tests/test-lib.sh
index 2368bd3..431231d 100644
--- a/tests/test-lib.sh
+++ b/tests/test-lib.sh
@@ -199,8 +199,9 @@ test_debug () {
 }
 
 test_run_ () {
-	eval >&3 2>&4 "$1"
+	eval > output 2>&1 "$1"
 	eval_ret="$?"
+	cat >&3 output
 	return 0
 }
 
@@ -594,9 +595,9 @@ test_todo_session () {
 	"")
 	    if [ ! -z "$cmd" ]; then
 		if [ $status = 0 ]; then
-		    test_expect_output "$1 $subnum" "$cmd > output"
+		    test_expect_output "$1 $subnum" "$cmd"
 		else
-		    test_expect_code_and_output "$status" "$1 $subnum" "$cmd > output"
+		    test_expect_code_and_output "$status" "$1 $subnum" "$cmd"
 		fi
 
 		subnum=$(($subnum + 1))
@@ -612,9 +613,9 @@ test_todo_session () {
     done
     if [ ! -z "$cmd" ]; then
 	if [ $status = 0 ]; then
-	    test_expect_output "$1 $subnum" "$cmd > output"
+	    test_expect_output "$1 $subnum" "$cmd"
 	else
-	    test_expect_code_and_output "$status" "$1 $subnum" "$cmd > output"
+	    test_expect_code_and_output "$status" "$1 $subnum" "$cmd"
 	fi
     fi
 }

From c31716af470ff9267b12965288e967aadecf5aad Mon Sep 17 00:00:00 2001
From: Ingo Karkat <swdev@ingo-karkat.de>
Date: Thu, 26 Jan 2012 12:15:26 +0100
Subject: [PATCH 4/4] test-report: Use color highlighting as in test-lib.

The test aggregate results are easy to miss when running the entire test suite via "make test", as the status of the last test case is highlighted, but the aggregate results appear in an unformatted, uncolored block of text.
Copy the say_color() function from test-lib.sh. (Sorry for the duplication, I found no simple way to import or share just this piece of functionality without adding much complexity.)
Successes, errors and broken summaries will now be highlighted in the appropriate colors (unless --no-color is given or output is not to a terminal), but only if the number is more than zero.
---
 tests/aggregate-results.sh | 65 ++++++++++++++++++++++++++++++++++----
 1 file changed, 59 insertions(+), 6 deletions(-)

diff --git a/tests/aggregate-results.sh b/tests/aggregate-results.sh
index d5bab75..ecc258f 100755
--- a/tests/aggregate-results.sh
+++ b/tests/aggregate-results.sh
@@ -1,4 +1,57 @@
-#!/bin/sh
+#!/bin/bash
+
+[ "x$TERM" != "xdumb" ] && (
+		export TERM &&
+		[ -t 1 ] &&
+		tput bold >/dev/null 2>&1 &&
+		tput setaf 1 >/dev/null 2>&1 &&
+		tput sgr0 >/dev/null 2>&1
+	) &&
+	color=t
+
+case "$1" in
+--no-color)
+	color=; shift ;;
+esac
+
+if test -n "$color"; then
+	say_color () {
+		(
+		export TERM
+		case "$1" in
+			error) tput bold; tput setaf 1;; # bold red
+			skip)  tput bold; tput setaf 2;; # bold green
+			pass)  tput setaf 2;;            # green
+			info)  tput setaf 3;;            # brown
+			*) test -n "$quiet" && return;;
+		esac
+		shift
+		printf "* %s" "$*"
+		tput sgr0
+		echo
+		)
+	}
+else
+	say_color() {
+		test -z "$1" && test -n "$quiet" && return
+		shift
+		echo "* $*"
+	}
+fi
+
+get_color()
+{
+	# Only use the supplied color if there are actually instances of that
+	# type, so that a clean test run does not distract the user by the
+	# appearance of the error highlighting.
+	if [ ${1:?} -eq 0 ]
+	then
+		echo 'info'
+	else
+		echo "${2:-info}"
+	fi
+}
+
 
 fixed=0
 success=0
@@ -27,8 +80,8 @@ do
 	done <"$file"
 done
 
-printf "%-8s%d\n" fixed $fixed
-printf "%-8s%d\n" success $success
-printf "%-8s%d\n" failed $failed
-printf "%-8s%d\n" broken $broken
-printf "%-8s%d\n" total $total
+say_color 'info'                           "$(printf "%-8s%d\n" fixed $fixed)"
+say_color "$(get_color "$success" 'pass')" "$(printf "%-8s%d\n" success $success)"
+say_color "$(get_color "$failed" 'error')" "$(printf "%-8s%d\n" failed $failed)"
+say_color "$(get_color "$broken" 'error')" "$(printf "%-8s%d\n" broken $broken)"
+say_color 'info'                           "$(printf "%-8s%d\n" total $total)"