Use the newer automake test driver "parallel-tests". This driver:

* keeps test logs around
* provides parallel testing functionality (with make -jN).

I've also added new functionality in test/Makefile.am which deletes old test logs, ensures that ALL tests are tried (even if units fail), and provides a color-coded summary at the end of the test run, which shows how many units, regressions (per level), and system tests failed (or passed), and provides a link to the log file for further information.

Resolves bug 117.

4 files changed, 52 insertions, 17 deletions

diff --git a/configure.ac b/configure.ac
index e7d5a005a..f805c445c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -413,7 +413,7 @@ if test "$enable_profiling" = yes; then
   CVC4LDFLAGS="${CVC4LDFLAGS:+$CVC4LDFLAGS }-pg"
 fi
 
-AM_INIT_AUTOMAKE([1.11 no-define color-tests])
+AM_INIT_AUTOMAKE([1.11 no-define parallel-tests color-tests])
 AC_CONFIG_HEADERS([cvc4autoconfig.h])
 
 # Initialize libtool's configuration options.
diff --git a/test/Makefile.am b/test/Makefile.am
index 316b2a140..31978b3c2 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -1,11 +1,46 @@
-SUBDIRS = unit system regress
+SUBDIRS = unit system regress .
+
+MAKEFLAGS = -k
 
 .PHONY: regress0 regress1 regress2 regress3
 regress0 regress1 regress2 regress3:
+	@$(MAKE) check-pre; \
 	for dir in $(SUBDIRS); do \
-		(cd $$dir && $(MAKE) $(AM_MAKEFLAGS) $@) || exit 1; \
-	done
+		test $$dir = . || (cd $$dir && $(MAKE) $(AM_MAKEFLAGS) $@); \
+	done; \
+	$(MAKE) check-local
 
 # synonyms for "check"
 .PHONY: regress test
 regress test: check
+
+am__tty_colors = \
+red=; grn=; lgn=; blu=; std=; \
+test "X$(AM_COLOR_TESTS)" != Xno \
+&& test "X$$TERM" != Xdumb \
+&& { test "X$(AM_COLOR_TESTS)" = Xalways || test -t 1 2>/dev/null; } \
+&& { \
+  red=''; \
+  grn=''; \
+  lgn=''; \
+  blu=''; \
+  std=''; \
+}
+subdirs_to_check = unit system regress/regress0 regress/regress1 regress/regress2 regress/regress3
+check-recursive: check-pre
+.PHONY: check-pre
+check-pre:
+	@rm -f $(subdirs_to_check:=/test-suite.log)
+check-local:
+	@$(am__tty_colors); \
+	echo $${blu}===============================  TESTING SUMMARY  =============================$$std; \
+	for log in $(subdirs_to_check:=/test-suite.log); do \
+		if test -s "$$log"; then \
+			status="`head -n 5 $$log | tail -1`"; \
+			if echo "$$status" | grep -q failed; then col=$$red; \
+			else col=$$grn; fi; \
+			echo "$$col$$status"; \
+			echo "    @abs_builddir@/$$log$$std"; \
+		fi; \
+	done; \
+	echo $${blu}===============================  TESTING SUMMARY  =============================$$std
diff --git a/test/regress/Makefile.am b/test/regress/Makefile.am
index 6ed661de9..13ed5b5f6 100644
--- a/test/regress/Makefile.am
+++ b/test/regress/Makefile.am
@@ -6,7 +6,7 @@ regress1: regress0
 regress2: regress0 regress1
 regress3: regress0 regress1 regress2
 regress0 regress1 regress2 regress3:
-	cd $@ && $(MAKE) check
+	-cd $@ && $(MAKE) check
 
 # synonyms for "check"
 .PHONY: regress test
diff --git a/test/unit/Makefile.am b/test/unit/Makefile.am
index fbca75676..ef60febf8 100644
--- a/test/unit/Makefile.am
+++ b/test/unit/Makefile.am
@@ -29,7 +29,7 @@ UNIT_TESTS = \
 	util/integer_black \
 	util/integer_white \
 	util/rational_black \
-    util/rational_white
+	util/rational_white
 
 # Things that aren't tests but that tests rely on and need to
 # go into the distribution
@@ -69,7 +69,7 @@ MOSTLYCLEANFILES = $(UNIT_TESTS) $(UNIT_TESTS:%=%.cpp)
 
 # We leave "TESTS" empty here; it's handled in Makefile.tests (see
 # that file for comment)
-# TESTS =
+TESTS = $(UNIT_TESTS)
 
 if STATIC_BINARY
 unit_LINK = $(CXXLINK) -all-static
@@ -83,39 +83,39 @@ $(UNIT_TESTS:%=@am__quote@./$(DEPDIR)/%.Plo@am__quote@): %.Plo:
 	$(AM_V_at)$(MKDIR_P) `dirname "$@"`
 	$(AM_V_GEN)test -e "$@" || touch "$@"
 
-$(UNIT_TESTS:%=%.cpp): %.cpp: %.h
+$(UNIT_TESTS:%=@abs_builddir@/%.cpp): @abs_builddir@/%.cpp: %.h
 	$(AM_V_at)$(MKDIR_P) `dirname "$@"`
 	$(AM_V_GEN)$(CXXTESTGEN) --have-eh --have-std --error-printer -o "$@" "$<"
 
-$(WHITE_TESTS:%=%.lo): %_white.lo: %_white.cpp
-@am__fastdepCXX_TRUE@	$(AM_V_CXX)$(LTCXXCOMPILE) $(AM_CXXFLAGS_WHITE) -MT $@ -MD -MP -MF $(DEPDIR)/$(@:%.lo=%).Tpo -c -o $@ @abs_builddir@/$<
+$(WHITE_TESTS:%=%.lo): %_white.lo: @abs_builddir@/%_white.cpp
+@am__fastdepCXX_TRUE@	$(AM_V_CXX)$(LTCXXCOMPILE) $(AM_CXXFLAGS_WHITE) -MT $@ -MD -MP -MF $(DEPDIR)/$(@:%.lo=%).Tpo -c -o $@ $<
 @am__fastdepCXX_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$(@:%.lo=%).Tpo $(DEPDIR)/$(@:%.lo=%).Plo
 @am__fastdepCXX_FALSE@	$(AM_V_CXX) @AM_BACKSLASH@
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@	source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
-@am__fastdepCXX_FALSE@	$(LTCXXCOMPILE) $(AM_CXXFLAGS_WHITE) -c -o $@ @abs_builddir@/$<
+@am__fastdepCXX_FALSE@	$(LTCXXCOMPILE) $(AM_CXXFLAGS_WHITE) -c -o $@ $<
 
 $(WHITE_TESTS): %_white: %_white.lo $(AM_LIBADD_WHITE)
 	$(AM_V_CXXLD)$(unit_LINK) $(AM_LIBADD_WHITE) $(AM_LDFLAGS_WHITE) $<
 
-$(BLACK_TESTS:%=%.lo): %_black.lo: %_black.cpp
-@am__fastdepCXX_TRUE@	$(AM_V_CXX)$(LTCXXCOMPILE) $(AM_CXXFLAGS_BLACK) -MT $@ -MD -MP -MF $(DEPDIR)/$(@:%.lo=%).Tpo -c -o $@ @abs_builddir@/$<
+$(BLACK_TESTS:%=%.lo): %_black.lo: @abs_builddir@/%_black.cpp
+@am__fastdepCXX_TRUE@	$(AM_V_CXX)$(LTCXXCOMPILE) $(AM_CXXFLAGS_BLACK) -MT $@ -MD -MP -MF $(DEPDIR)/$(@:%.lo=%).Tpo -c -o $@ $<
 @am__fastdepCXX_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$(@:%.lo=%).Tpo $(DEPDIR)/$(@:%.lo=%).Plo
 @am__fastdepCXX_FALSE@	$(AM_V_CXX) @AM_BACKSLASH@
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@	source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
-@am__fastdepCXX_FALSE@	$(LTCXXCOMPILE) $(AM_CXXFLAGS_BLACK) -c -o $@ @abs_builddir@/$<
+@am__fastdepCXX_FALSE@	$(LTCXXCOMPILE) $(AM_CXXFLAGS_BLACK) -c -o $@ $<
 
 $(BLACK_TESTS): %_black: %_black.lo $(AM_LIBADD_BLACK)
 	$(AM_V_CXXLD)$(unit_LINK) $(AM_LIBADD_BLACK) $(AM_LDFLAGS_BLACK) $<
 
-$(PUBLIC_TESTS:%=%.lo): %_public.lo: %_public.cpp
-@am__fastdepCXX_TRUE@	$(AM_V_CXX)$(LTCXXCOMPILE) $(AM_CXXFLAGS_PUBLIC) -MT $@ -MD -MP -MF $(DEPDIR)/$(@:%.lo=%).Tpo -c -o $@ @abs_builddir@/$<
+$(PUBLIC_TESTS:%=%.lo): %_public.lo: @abs_builddir@/%_public.cpp
+@am__fastdepCXX_TRUE@	$(AM_V_CXX)$(LTCXXCOMPILE) $(AM_CXXFLAGS_PUBLIC) -MT $@ -MD -MP -MF $(DEPDIR)/$(@:%.lo=%).Tpo -c -o $@ $<
 @am__fastdepCXX_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$(@:%.lo=%).Tpo $(DEPDIR)/$(@:%.lo=%).Plo
 @am__fastdepCXX_FALSE@	$(AM_V_CXX) @AM_BACKSLASH@
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@	source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
-@am__fastdepCXX_FALSE@	$(LTCXXCOMPILE) $(AM_CXXFLAGS_PUBLIC) -c -o $@ @abs_builddir@/$<
+@am__fastdepCXX_FALSE@	$(LTCXXCOMPILE) $(AM_CXXFLAGS_PUBLIC) -c -o $@ $<
 
 $(PUBLIC_TESTS): %_public: %_public.lo $(AM_LIBADD_PUBLIC)
 	$(AM_V_CXXLD)$(unit_LINK) $(AM_LIBADD_PUBLIC) $(AM_LDFLAGS_PUBLIC) $<