tools-make/TestFramework/gnustep-tests

#!/bin/sh
#
#  Runs tests for the GNUstep Testsuite
#
#  Copyright (C) 2005-2011 Free Software Foundation, Inc.
#
#  Written by:  Alexander Malmberg <alexander@malmberg.org>
#  Updates by:  Richard Frith-Macdonald <rfm@gnu.org>
#
#  This package is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public
#  License as published by the Free Software Foundation; either
#  version 3 of the License, or (at your option) any later version.
#
#  This library is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  General Public License for more details.
#
#
# Usage: gnustep-tests [directory | test1.m [test2.m ...]]
#
# Runs the tests in the specified directory (or those in the individual files)
# or all the tests in subdirectories of the current directory if no arguments
# are given.
# A summary is written to tests.sum, a log to tests.log, and a brief
# summary to stdout.
# The log and summary from the previous testrun are renamed to
# oldtests.log and oldtests.sum, available for comparison.

if test -z "$GNUSTEP_MAKEFILES"; then
  GNUSTEP_MAKEFILES=`gnustep-config --variable=GNUSTEP_MAKEFILES 2>/dev/null`
  if test -z "$GNUSTEP_MAKEFILES"; then
    echo "You need to have GNUstep-make installed and set up."
    echo "Did you remember to source GNUstep.sh?"
  else
    echo "You forgot to set your GNUSTEP_MAKEFILES environment variable."
    echo "Setting it to $GNUSTEP_MAKEFILES during this test run."
    export GNUSTEP_MAKEFILES
    . $GNUSTEP_MAKEFILES/GNUstep.sh
  fi
fi

GSTESTMODE=normal

# Argument checking
while test $# != 0
do
  gs_option=
  case $1 in
    --clean)
      GSTESTMODE=clean
      ;;
    --documentation)
      echo
      echo "$0: Script to run the GNUstep testsuite"
      echo "Usage: gnustep-tests [directory | test1.m [test2.m ...]]"
      echo "Runs the specified tests, or any in subdirectories of the"
      echo "current directory if no arguments are given."
      echo "Use 'gnustep-tests --help' for basic help."
      echo
      cat $GNUSTEP_MAKEFILES/TestFramework/README
      exit 0
      ;;
    --failfast)
      GSTESTMODE=failfast
      ;;
    --help | -h)
      echo
      echo "$0: Script to run the GNUstep testsuite"
      echo "Usage: gnustep-tests [directory | test1.m [test2.m ...]]"
      echo "Runs the specified tests, or any in subdirectories of the"
      echo "current directory if no arguments are given."
      echo "Use 'gnustep-tests --documentation' for full details."
      echo "Use 'gnustep-tests --failfast' to stop at the first failure."
      echo "Use 'gnustep-tests --clean' to remove old logs and leftover files."
      echo
      echo "Interpreting the output"
      echo "-----------------------"
      echo "The summary output lists all test failures ... there should not"
      echo "be any.  If a test fails then either there is a problem in the"
      echo "software being tested, or a problem in the test itself. Either"
      echo "way, you should try to fix the problem and provide a patch, or"
      echo "at least report it at: https://savannah.gnu.org/bugs/?group=gnustep"
      echo
      exit 0
      ;;
      --debug | -d)	# ignore for backward compatibility.
      ;;
    *)
      break
      ;;
  esac
  shift
done

if [ ! "$MAKE_CMD" ]
then
  gmake --version > /dev/null 2>&1
  if [ $? = 0 ]
  then
    MAKE_CMD=gmake
  else
    MAKE_CMD=make
  fi
fi
export MAKE_CMD

if test $# = 0
then
  echo "Checking for presence of test subdirectories ..."
fi

TEMP=`echo *`
TESTDIRS=
for file in $TEMP
do
  if [ -d $file -a $file != CVS -a $file != obj ]
  then
    TESTDIRS="$TESTDIRS $file"
  fi
done

if [ x$1 != x ]
then
  if [ -d $1 ]
  then
    # Only find in the directories specified.
    TESTDIRS=$*
  else
    TESTDIRS=
    TESTS=$*
  fi
fi

CWD=`pwd`
OWD=
RUNCMD=$GNUSTEP_MAKEFILES/TestFramework/runtest.sh
RUNEXIT=0

run_test_file ()
{
  sub=`dirname $TESTFILE`
  if [ "x$OWD" != "x$sub" ]
  then
    OWD=$sub
    if [ "$GSTESTMODE" = "clean" ]
    then
      echo "--- Cleaning tests in $sub ---"
      rm -rf $sub/GNUmakefile.tmp $sub/obj $sub/core
      rm -rf $sub/tests.tmp $sub/tests.sum.tmp
      rm -rf $sub/tests.log $sub/tests.sum
      rm -rf $sub/oldtests.log $sub/oldtests.sum
    else
      echo "--- Running tests in $sub ---"
      echo "--- Running tests in $sub ---" >> $CWD/tests.log
      if [ -r $dir/URGENT ]
      then
	cat $dir/URGENT
	cat $dir/URGENT >> $CWD/tests.log
      fi
      if [ -r $dir/README ]
      then
	cat $dir/README >> $CWD/tests.log
      fi
    fi
  fi

  if [ "$GSTESTMODE" != "clean" ]
  then
    echo >> $CWD/tests.log
    echo Testing $TESTFILE... >> $CWD/tests.log
    echo >> $CWD/tests.sum

    # Run the test. Log everything to a temporary file.
    export GSTESTMODE
    $RUNCMD $run_args $TESTFILE > $CWD/tests.tmp 2>&1
    RUNEXIT=$?

    # Add the information to the detailed log.
    cat $CWD/tests.tmp >> $CWD/tests.log

    # Extract the summary information and add it to the summary file.
    grep "^\(Passed test\|Failed test\|Failed build\|Completed file\|Failed file\|Dashed hope\|Failed set\|Skipped set\):" $CWD/tests.tmp > $CWD/tests.sum.tmp
    cat $CWD/tests.sum.tmp >> $CWD/tests.sum

    # If there were failures or unresolved tests then report them...
    if grep -L "^\(Failed build\|Failed file\|Failed set\|Failed test\):" $CWD/tests.sum.tmp > /dev/null; then
      echo
      echo $TESTFILE:
      grep "^\(Failed build\|Failed file\|Failed set\|Failed test\):" $CWD/tests.sum.tmp
    fi
  fi
}


# Replace the old files.
if [ -f tests.log ]
then
  mv tests.log oldtests.log
fi
if [ -f tests.sum ]
then
  mv tests.sum oldtests.sum
fi

if [ x"$TESTDIRS" = x ]
then
  # Run specific individual test files.
  for TESTFILE in $TESTS
  do
    run_test_file
    if [ "$RUNEXIT" != "0" ]
    then
      break
    fi
  done
else
  for dir in $TESTDIRS
  do
    TESTS=`find $dir -name \*.m | sort | sed -e 's/\(^\| \)X[^ ]*//g'`

    # If there are no test files found, we need to print out a message
    # at this level to let people know we processed the directory.
    if [ "x$TESTS" = "x" ]
    then
      if [ "$GSTESTMODE" = "clean" ]
      then
        echo "--- Cleaning tests in $dir ---"
      else
        echo "--- Running tests in $dir ---"
        echo "--- Running tests in $dir ---" >> $CWD/tests.log
        if [ -r $dir/URGENT ]
	then
	  cat $dir/URGENT
	  cat $dir/URGENT >> $CWD/tests.log
	fi
        if [ -r $dir/README ]
        then
	  cat $dir/README >> $CWD/tests.log
        fi
      fi
    fi

    # If there is a GNUmakefile.tests in the directory, run it first.
    cd $dir
    if [ -f GNUmakefile.tests ]
    then
      $MAKE_CMD -f GNUmakefile.tests $MAKEFLAGS debug=yes 2>&1
    fi

    # Now we process each test file in turn.
    cd $CWD
    for TESTFILE in $TESTS
    do
      run_test_file
      if [ "$RUNEXIT" != "0" ]
      then
        break
      fi
    done
  done
fi

if [ "$GSTESTMODE" = "clean" ]
then
  rm -f tests.tmp tests.sum.tmp
  rm -f tests.log tests.sum
  rm -f oldtests.log oldtests.sum
else
  # Make some stats.
  if [ -r tests.sum ]
  then
    # Nasty pipeline of commands ...
    # Look for each type of test result, sort and count the results,
    # append 's' to each summary, then remove the trailing 's' from
    # any summary with only a single result so the output is pretty.
    # Sort the resulting lines by number of each status with the most
    # common (hopefully passes) output first.
    # NB. we omit the 'Completed file' tests as uninteresting ... users
    # generally only want to see the total pass count and any problems.
    grep "^\(Passed test\|Failed test\|Failed build\|Failed file\|Dashed hope\|Failed set\|Skipped set\):" tests.sum | cut -d: -f1 | sort | uniq -c | sed -e 's/.*/&s/' | sed -e 's/^\([^0-9]*1[^0-9].*\)s$/\1/' | sort -n -b -r > tests.tmp
  else
    echo "No tests found." > tests.tmp
  fi

  echo >> tests.sum
  cat tests.tmp >> tests.sum

  echo
  cat tests.tmp
  echo

  grep -q "\(Failed set\|Failed sets\|Failed test\|Failed tests\|Failed build\|Failed build\|Failed file\|Failed files\)$" tests.tmp
  if [ $? = 1 ]
  then
    echo "All OK!"

    grep -q "\(Dashed hope\|Dashed hopes\)$" tests.tmp
    if [ $? = 0 ]
    then
      echo 
      echo "But we were hoping that even more tests might have passed if"
      echo "someone had added support for them to the package.  If you"
      echo "would like to help, please contact the package maintainer."
    fi

    grep -q "\(Skipped set\|Skipped sets\)$" tests.tmp
    if [ $? = 0 ]
    then
      echo 
      echo "Even though no tests failed, we had to skip some testing"
      echo "due to lack of support on your system.  This might be because"
      echo "some required software library was just not available when the"
      echo "software was built (in which case you can install that library"
      echo "and rebuild, then re-run the tests), or the required functions"
      echo "may not be available on your operating system at all."
      echo "If you would like to contribute code to add the missing"
      echo "functionality, please contact the package maintainer."
    fi

  else
    if [ "$GSTESTMODE" = "failfast" ]
    then
      exit 0
    fi

    grep -q "\(Failed build\|Failed build\)$" tests.tmp
    if [ $? = 0 ]
    then
      echo
      echo "Unfortunately we could not even compile all the test programs."
      echo "This means that the test could not be run properly, and you need"
      echo "to try to figure out why and fix it or ask for help."
    fi

    grep -q "\(Failed file\|Failed files\)$" tests.tmp
    if [ $? = 0 ]
    then
      echo
      echo "Some testing was abandoned when a test program aborted.  This is"
      echo "generally a severe problem and may nean that the package is"
      echo "completely unusuable.  You need to try to fix this and, if it's"
      echo "not due to some problem on your system, please help by submitting"
      echo "a patch (or at least a bug report) to the package maintainer."
    fi

    grep -q "\(Failed set\|Failed sets\)$" tests.tmp
    if [ $? = 0 ]
    then
      echo
      echo "Some set of tests failed.  This could well mean that a large"
      echo "number of individual tests dis not pass and that there are"
      echo "severe problems in the software."
      echo "Please submit a patch to fix the problem or send a bug report to"
      echo "the package maintainer."
    fi

    grep -q "\(Failed test\|Failed tests\)$" tests.tmp
    if [ $? = 0 ]
    then
      echo
      echo "One or more tests failed.  None of them should have."
      echo "Please submit a patch to fix the problem or send a bug report to"
      echo "the package maintainer."
    fi
  fi

  echo
fi

# Delete the temporary file.
rm -f tests.tmp tests.sum.tmp