From 265d52abcb4181e06cf5a60465b0a9514ecc4be9 Mon Sep 17 00:00:00 2001 From: Joseph Myers Date: Fri, 7 Mar 2014 03:25:57 +0000 Subject: Generate overall summary of test results. This patch, an updated version of , makes testsuite runs generate an overall summary of test results. A new script merge-test-results.sh deals both with collecting results within a directory to a file with all the results from that directory, and collecting the results from subdirectories into a single overall file (there's not much in common between the two modes of operation of the script, but it seemed silly to have two separate scripts for this). Within a directory, missing results produce UNRESOLVED lines; at top level, missing results for a whole directory produce an ERROR line (since toplevel can't identify what the specific missing tests are in this case). Note that this does not change the rules for when "make" considers there has been an error, or terminates, so unexpected failures will still cause make to terminate, or, with -k, mean the commands for "tests" don't get run because of failure of a dependency. Tested x86_64, including that the summary does in fact reflect all the tests with .test-result files. * scripts/merge-test-results.sh: New file. * Makefile (tests-special-notdir): New variable. (tests): Run merge-test-results.sh. (xtests): Likewise. * Rules (tests-special-notdir): New variable. (xtests-special-notdir): Likewise. (tests): Run merge-test-results.sh (xtests): Likewise. --- ChangeLog | 9 +++++++ Makefile | 11 ++++++++ Rules | 11 ++++++++ scripts/merge-test-results.sh | 61 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+) create mode 100755 scripts/merge-test-results.sh diff --git a/ChangeLog b/ChangeLog index bb48f748b5..ae5c05592a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,14 @@ 2014-03-07 Joseph Myers + * scripts/merge-test-results.sh: New file. + * Makefile (tests-special-notdir): New variable. + (tests): Run merge-test-results.sh. + (xtests): Likewise. + * Rules (tests-special-notdir): New variable. + (xtests-special-notdir): Likewise. + (tests): Run merge-test-results.sh + (xtests): Likewise. + * Makeconfig (test-xfail-name): New variable. (evaluate-test): Use $(test-xfail-name) instead of $(@F:.out=) to compute variable name for expected failures. diff --git a/Makefile b/Makefile index aee88e1e3e..8214dda840 100644 --- a/Makefile +++ b/Makefile @@ -316,7 +316,18 @@ $(objpfx)begin-end-check.out: scripts/begin-end-check.pl $(evaluate-test) endif +tests-special-notdir = $(patsubst $(objpfx)%, %, $(tests-special)) tests: $(tests-special) + $(..)scripts/merge-test-results.sh -s $(objpfx) "" \ + $(sort $(tests-special-notdir:.out=)) \ + > $(objpfx)subdir-tests.sum + $(..)scripts/merge-test-results.sh -t $(objpfx) subdir-tests.sum \ + $(sort $(subdirs) .) \ + > $(objpfx)tests.sum +xtests: + $(..)scripts/merge-test-results.sh -t $(objpfx) subdir-xtests.sum \ + $(sort $(subdirs)) \ + > $(objpfx)xtests.sum # The realclean target is just like distclean for the parent, but we want # the subdirs to know the difference in case they care. diff --git a/Rules b/Rules index d37ae52b6c..feb304dbc0 100644 --- a/Rules +++ b/Rules @@ -102,6 +102,17 @@ tests: $(tests:%=$(objpfx)%.out) $(tests-special) xtests: tests $(xtests:%=$(objpfx)%.out) $(xtests-special) endif +tests-special-notdir = $(patsubst $(objpfx)%, %, $(tests-special)) +xtests-special-notdir = $(patsubst $(objpfx)%, %, $(xtests-special)) +tests: + $(..)scripts/merge-test-results.sh -s $(objpfx) $(subdir) \ + $(sort $(tests) $(tests-special-notdir:.out=)) \ + > $(objpfx)subdir-tests.sum +xtests: + $(..)scripts/merge-test-results.sh -s $(objpfx) $(subdir) \ + $(sort $(xtests) $(xtests-special-notdir:.out=)) \ + > $(objpfx)subdir-xtests.sum + ifeq ($(build-programs),yes) binaries-all-notests = $(others) $(sysdep-others) binaries-all-tests = $(tests) $(xtests) $(test-srcs) diff --git a/scripts/merge-test-results.sh b/scripts/merge-test-results.sh new file mode 100755 index 0000000000..6255e242b1 --- /dev/null +++ b/scripts/merge-test-results.sh @@ -0,0 +1,61 @@ +#! /bin/sh +# Merge test results of individual tests or subdirectories. +# Copyright (C) 2014 Free Software Foundation, Inc. +# This file is part of the GNU C Library. + +# The GNU C Library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. + +# The GNU C Library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. + +# You should have received a copy of the GNU Lesser General Public +# License along with the GNU C Library; if not, see +# . + +# usage: merge-test-results.sh -s objpfx subdir test-name... +# (subdirectory tests; empty subdir at top level), or +# merge-test-results.sh -t objpfx subdir-file-name subdir... +# (top-level merge) + +set -e + +type=$1 +objpfx=$2 +shift 2 + +case $type in + -s) + subdir=$1 + shift + subdir=${subdir:+$subdir/} + for t in "$@"; do + if [ -s "$objpfx$t.test-result" ]; then + head -n1 "$objpfx$t.test-result" + else + echo "UNRESOLVED: $subdir$t" + fi + done + ;; + + -t) + subdir_file_name=$1 + shift + for d in "$@"; do + if [ -f "$objpfx$d/$subdir_file_name" ]; then + cat "$objpfx$d/$subdir_file_name" + else + echo "ERROR: test results for $d directory missing" + fi + done + ;; + + *) + echo "unknown type $type" >&2 + exit 1 + ;; +esac -- cgit v1.2.3