#!@SHELLPROG@
#
#  Runs tests for the GNUstep Testsuite
#
#  Copyright (C) 2005-2011 Free Software Foundation, Inc.
#
#  Written by:  Alexander Malmberg <alexander@malmberg.org>
#  Updates by:  Richard Frith-Macdonald <rfm@gnu.org>
#
#  This package is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public
#  License as published by the Free Software Foundation; either
#  version 3 of the License, or (at your option) any later version.
#
#  This library is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  General Public License for more details.
#
#
# Usage: gnustep-tests [directory | test.m]
#
# Runs the tests in the specified directory (or those in the individual file)
# or all the tests in subdirectories of the current directory if no arguments
# are given.
# A summary is written to tests.sum, a log to tests.log, and a brief
# summary to stdout.
# The log and summary from the previous testrun are renamed to
# oldtests.log and oldtests.sum, available for comparison.

if test -z "$GNUSTEP_MAKEFILES"; then
  GNUSTEP_MAKEFILES=`gnustep-config --variable=GNUSTEP_MAKEFILES 2>/dev/null`
  if test -z "$GNUSTEP_MAKEFILES"; then
    echo "You need to have GNUstep-make installed and set up."
    echo "Did you remember to source GNUstep.sh?"
  else
    echo "You forgot to set your GNUSTEP_MAKEFILES environment variable."
    echo "Setting it to $GNUSTEP_MAKEFILES during this test run."
    export GNUSTEP_MAKEFILES
    . $GNUSTEP_MAKEFILES/GNUstep.sh
  fi
fi

GSTESTTOP="$GNUSTEP_MAKEFILES/TestFramework"
export GSTESTTOP
GSTESTDIR=`pwd`
export GSTESTDIR

GSTESTMODE=normal

# Argument checking
while test $# != 0
do
  gs_option=
  case $1 in
    --clean)
      GSTESTMODE=clean
      ;;
      --debug)
      GSTESTDBG="$GSTESTDIR/gdb.cmds"
      ;;
    --documentation)
      echo
      echo "$0: Script to run the GNUstep testsuite"
      echo "Usage: gnustep-tests [directory | test1.m [test2.m ...]]"
      echo "Runs the specified tests, or any in subdirectories of the"
      echo "current directory if no arguments are given."
      echo "Use 'gnustep-tests --help' for basic help."
      echo
      cat $GSTESTTOP/README
      exit 0
      ;;
    --verbose)
      GSVERBOSE=yes
      ;;
    --failfast)
      GSTESTMODE=failfast
      ;;
    --help | -h)
      echo
      echo "$0: Script to run the GNUstep testsuite"
      echo "Usage: gnustep-tests [directory | test.m]]"
      echo "Runs the specified tests, or any in subdirectories of the"
      echo "current directory if no arguments are given."
      echo "Use 'gnustep-tests --documentation' for full details."
      echo "Use 'gnustep-tests --clean' to remove old logs and leftover files."
      echo "Use 'gnustep-tests --failfast' to stop after the first failure."
      echo "Use 'gnustep-tests --debug' to run gdb for any failed tests."
      echo "Use 'gnustep-tests --verbose' for full/detailed log output."
      echo
      echo "Interpreting the output"
      echo "-----------------------"
      echo "The summary output lists all test failures ... there should not"
      echo "be any.  If a test fails then either there is a problem in the"
      echo "software being tested, or a problem in the test itself. Either"
      echo "way, you should try to fix the problem and provide a patch, or"
      echo "at least report it at: https://savannah.gnu.org/bugs/?group=gnustep"
      echo
      exit 0
      ;;
    *)
      break
      ;;
  esac
  shift
done

export GSTESTMODE
GSTESTLOG=$GSTESTDIR/tests.log
export GSTESTLOG
GSTESTSUM=$GSTESTDIR/tests.sum
export GSTESTSUM

if test x"$CC" = x
then
  CC=`gnustep-config --variable=CC`
  export CC
fi
GSTESTFLAGS=`gnustep-config --objc-flags`
GSTESTFLAGS="$GSTESTFLAGS -I$GSTESTTOP"
if test "$GSTESTMODE" = "failfast"
then
  GSTESTFLAGS="$GSTESTFLAGS -DFAILFAST=1"
fi
GSTESTLIBS=`gnustep-config --gui-libs`

if test x"$BASH_VERSION" = x
then
# In some shells the built in test command actually only implements a subset
# of the normally expected functionality (or is partially broken), so we
# define a function to call a real program to do the job.
test()
{
  @TESTPROG@ $@
}
fi

if test ! "$MAKE_CMD"
then
  MAKE_CMD=`gnustep-config --variable=GNUMAKE`
  $MAKE_CMD --version > /dev/null 2>&1
  if test $? != 0
  then
    MAKE_CMD=gmake
    $MAKE_CMD --version > /dev/null 2>&1
    if test $? != 0
    then
      MAKE_CMD=make
    fi
  fi
fi

if test $# = 0
then
  echo "Checking for presence of test subdirectories ..."
fi

TEMP=`echo *`
TESTS=
TESTDIRS=
for file in $TEMP
do
  if test -d $file -a $file != CVS -a $file != obj
  then
    TESTDIRS="$TESTDIRS $file"
  fi
done

if test x$1 != x
then
  if test -d $1
  then
    # Only find in the directories specified.
    TESTDIRS=$*
  elif test -r $1
  then
    TESTDIRS=`dirname $1`
    TESTS=`basename $1`
    BARE=`basename $TESTS .m`
    if test x"$BARE" = x"$TESTS"
    then
      echo "The file '$1' does not have a .m extension ... cannot test."
      exit 1
    fi
  else
    echo "'$1' is not a directory or a readable source file ... cannot test."
    exit 1
  fi
fi

# Function for platforms where grep can't search for multiple patterns.
extract()
{
  f=$1
  shift
  while test $# != 0
  do
    grep "$1" "$f"
    shift
  done
}

# Function for platforms where grep can't search for multiple patterns.
present()
{
  f=$1
  shift
  while test $# != 0
  do
    grep "$1" "$f" >/dev/null
    if test $? = "0"
    then
      return 0
    fi
    shift
  done
  return 1
}

# Low level function to build and run the Objective-C program $TESTFILE
# in the current directory.  The TEMPLATE variable must already be set
# to the name of the make file template if gnustep-make is to do the
# building.
# 
build_and_run ()
{
  # Remove the extension, if there is one. If there is no extension, add
  # .obj .
  TESTNAME=`echo $TESTFILE | sed -e"s/^\(test^.]*\)$/\1.obj./;s/\.[^.]*//g"`

  # Run the test.

  RUN_CMD="./obj/$TESTNAME"
  if test x"$TEMPLATE" = x
  then
    # The very simple case, we just need to compile a single file
    # putting the executable in the obj subdirectory.
    rm -rf ./obj
    mkdir ./obj
    BUILD_CMD="$CC -o ./obj/$TESTNAME $TESTFILE $GSTESTFLAGS $GSTESTLIBS"
    CLEAN_CMD=echo
  else
    BUILD_CMD="$MAKE_CMD $MAKEFLAGS debug=yes"
    CLEAN_CMD="$MAKE_CMD clean"
    # Create the GNUmakefile by filling in the name of the test,
    # the name of the file, the include directory, and the failfast
    # option if needed.
    rm -f GNUmakefile
    TESTRULES="${TESTNAME}_OBJC_FILES=$TESTFILE"
    if test "$GSTESTMODE" = "failfast"
    then
      sed -e "s/@TESTNAMES@/$TESTNAME/;s/@TESTRULES@/$TESTRULES/;s^@TESTFLAGS@^-I$GSTESTTOP -DFAILFAST=1^" < "$TEMPLATE" > GNUmakefile
    else
      sed -e "s/@TESTNAMES@/$TESTNAME/;s/@TESTRULES@/$TESTRULES/;s^@TESTFLAGS@^-I$GSTESTTOP^" < "$TEMPLATE" > GNUmakefile
    fi
  fi

  if test "$GSTESTMODE" = "clean"
  then
    $CLEAN_CMD >/dev/null 2>&1
    rm -f GNUmakefile
    rm -rf obj core gdb.cmds
    return 0
  fi

  # Clean up to avoid contamination by previous tests. Assume
  # that this will never fail in any interesting way.
  $CLEAN_CMD >/dev/null 2>&1

  # Compile it. Redirect errors to stdout so it shows up in the log,
  # but not in the summary.
  $BUILD_CMD >$GSTESTLOG.tmp 2>&1
  if test $? != 0
  then
    echo "Failed build:     $1" >&2
    if test "$GSTESTMODE" = "failfast"
    then
      return 1
    fi
  else
    # We want aggressive memory checking.

    # Tell glibc to check for malloc errors, and to crash if it detects
    # any.
    MALLOC_CHECK_=2
    export MALLOC_CHECK

    # Tell GNUstep-base to check for messages sent to deallocated objects
    # and crash if it happens.
    NSZombieEnabled=YES
    CRASH_ON_ZOMBIE=YES
    export NSZombieEnabled CRASH_ON_ZOMBIE

    echo Running $dir/$TESTFILE...
    # Run it. If it terminates abnormally, mark it as a crash (unless we have
    # a special file to mark it as being expected to abort).
    $RUN_CMD
    if test $? != 0
    then
      if test -r $TESTFILE.abort
      then
	echo "Completed file:  $TESTFILE" >&2
      else
	echo "Failed file:     $TESTFILE aborted without running all tests!" >&2
	if test "$GSTESTMODE" = "failfast"
	then
	  return 1
	fi
      fi
    else
      echo "Completed file:  $TESTFILE" >&2
    fi
  fi
  return 0
}

# Function to build and run $TESTFILE in the current directory.
# This actually manages the logging process and calls build_and_run
# to perform the work.
#
run_test_file ()
{
  RUNEXIT=0
  if test "$GSTESTMODE" = "clean"
  then
    # Clean the test.
    build_and_run > /dev/null 2>&1
    return 0
  else
    echo >> $GSTESTLOG
    echo Testing $TESTFILE... >> $GSTESTLOG
    echo >> $GSTESTSUM

    if test x"$GSVERBOSE" = xyes
    then
      build_and_run 2>&1 | tee $GSTESTLOG.tmp
    else
      build_and_run > $GSTESTLOG.tmp 2>&1
    fi
    RUNEXIT=$?

    # Add the information to the detailed log.
    cat $GSTESTLOG.tmp >> $GSTESTLOG

    # Extract the summary information and add it to the summary file.
    extract $GSTESTLOG.tmp "^Passed test:" "^Failed test:" "^Failed build:" "^Completed file:" "^Failed file:" "^Dashed hope:" "^Failed set:" "^Skipped set:" > $GSTESTSUM.tmp
    cat $GSTESTSUM.tmp >> $GSTESTSUM

    # If there were failures or skipped tests then report them...
    if present $GSTESTSUM.tmp "^Failed build:" "^Failed file:" "^Failed set:" "^Failed test:" "^Skipped set:"
    then
      echo
      echo $dir/$TESTFILE:
      extract $GSTESTSUM.tmp "^Failed build:" "^Failed file:" "^Failed set:" "^Failed test:" "^Skipped set:"
    fi

    if test x"$GSTESTDBG" != x
    then
      if present "$GSTESTSUM.tmp" "^Failed test:"
      then
        grep '^Failed test:' "$GSTESTLOG.tmp" | sed -e 's/^Failed test:[^:]*:\([0-9][0-9]*\).*/break testStart if testLineNumber==\1/' > "$GSTESTDBG"
        gdb "./obj/$TESTNAME" -x "$GSTESTDBG"
        rm -f "$GSTESTDBG"
      fi
    fi

    return $RUNEXIT
  fi
}


# Replace the old files.
if test -f tests.log
then
  mv tests.log oldtests.log
fi
if test -f tests.sum
then
  mv tests.sum oldtests.sum
fi

SUMD=.
for TESTDIR in $TESTDIRS
do
  found=no
  # Get the names of all subdirectories containing source files.
  SRCDIRS=`find $TESTDIR -name \*.m | sed -e 's;/[^/]*$;;' | sort -u | sed -e 's/\(^\| \)X[^ ]*//g'`
  if test x"$SRCDIRS" = x
  then
    continue
  fi
  SUMD=$TESTDIR
  for dir in $SRCDIRS
  do
    if test ! -f $dir/TestInfo
    then
      continue
    fi
    found=yes
    cd $dir

    if test "$GSTESTMODE" = "clean"
    then
      echo "--- Cleaning tests in $dir ---"
    else
      echo "--- Running tests in $dir ---"
      if test -r ./Start.sh -a -x ./Start.sh
      then
	./Start.sh
      fi
    fi

    if test x"$TESTS" = x
    then
      TESTS=`echo *.m | sort | sed -e 's/\(^\| \)X[^ ]*//g'`
    fi
    # If there is a GNUmakefile.tests in the directory, run it first.
    if test -f GNUmakefile.tests
    then
      if test "$GSTESTMODE" = "clean"
      then
	$MAKE_CMD -f GNUmakefile.tests $MAKEFLAGS clean 2>&1
      else
	$MAKE_CMD -f GNUmakefile.tests $MAKEFLAGS debug=yes 2>&1
      fi
    fi

    if test -r GNUmakefile.tests 
    then
      TEMPLATE=GNUmakefile.tests
    elif test -r GNUmakefile.preamble 
    then
      TEMPLATE=$GSTESTTOP/GNUmakefile.in
    elif test -r GNUmakefile.postamble
    then
      TEMPLATE=$GSTESTTOP/GNUmakefile.in
    elif test -r ../GNUmakefile.super
    then
      TEMPLATE=$GSTESTTOP/GNUmakefile.in
    else
      TEMPLATE=
    fi

    # Now we process each test file in turn.
    # When cleaning, we only need to do one clean per directory.
    RUNEXIT=0
    for TESTFILE in $TESTS
    do
      run_test_file
      if test "$RUNEXIT" != "0" -o "$GSTESTMODE" = "clean"
      then
	break
      fi
    done
    TESTS=

    if test "$GSTESTMODE" = "clean"
    then
      rm -rf core obj GNUmakefile gdb.cmds
      rm -f tests.tmp tests.sum.tmp tests.log.tmp
      rm -f tests.log tests.sum
      rm -f oldtests.log oldtests.sum
    else
      # And perform the cleanup script.
      if test -r ./End.sh -a -x ./End.sh
      then
	./End.sh
      fi
    fi

    cd $GSTESTDIR
    if test "$RUNEXIT" != "0"
    then
      break
    fi
  done
  if test $found = no
  then
    echo "No tests found in $TESTDIR"
  fi
  if test "$RUNEXIT" != "0"
  then
    break
  fi
done

if test "$GSTESTMODE" = "clean"
then
  rm -rf core obj GNUmakefile.tmp gdb.cmds
  rm -f tests.tmp tests.sum.tmp tests.log.tmp
  rm -f tests.log tests.sum
  rm -f oldtests.log oldtests.sum
else
  # Make some stats.
  if test -r tests.sum
  then
    # Nasty pipeline of commands ...
    # Look for each type of test result, sort and count the results,
    # append 's' to each summary, then remove the trailing 's' from
    # any summary with only a single result so the output is pretty.
    # Sort the resulting lines by number of each status with the most
    # common (hopefully passes) output first.
    # NB. we omit the 'Completed file' tests as uninteresting ... users
    # generally only want to see the total pass count and any problems.
    extract tests.sum "^Passed test:" "^Failed test:" "^Failed build:" "^Failed file:" "^Dashed hope:" "^Failed set:" "^Skipped set:" | cut -d: -f1 | sort | uniq -c | sed -e 's/.*/&s/' | sed -e 's/^\([^0-9]*1[^0-9].*\)s$/\1/' | sort -n -b -r > tests.tmp
  else
    echo "No tests found." > tests.tmp
  fi

  echo >> tests.sum
  cat tests.tmp >> tests.sum

  echo
  cat tests.tmp
  echo

fi

# In the case where we ran a single testsuite, we allow the Summary.sh
# script in that testsuite to generate our summary.
if test x"$TESTDIRS" = x"$SUMD" -a -r $SUMD/Summary.sh -a -x $SUMD/Summary.sh
then
  RUNCMD=$SUMD/Summary.sh
else
  RUNCMD=$GSTESTTOP/Summary.sh
fi
$RUNCMD

# Delete the temporary file.
rm -f tests.tmp tests.sum.tmp tests.log.tmp