Add --developer mode and document

git-svn-id: svn+ssh://svn.gna.org/svn/gnustep/tools/make/trunk@32565 72102866-910b-0410-8b05-ffd578937521

ChangeLog

@@ -1,3 +1,9 @@
+2011-03-14  Richard Frith-Macdonald <rfm@gnu.org>
+
+	* TestFramework/gnustep-tests.in: Add --developer option
+	* TestFramework/Testing.h: Make --developer turn off hopes
+	* TestFramework/README: Document --developer and GSTESTROOT
+
 2011-03-13  Nicola Pero  <nicola.pero@meta-innovation.com>
 
 	* configure.ac: Always use 'fhs' layout as the default layout

TestFramework/README

@@ -123,11 +123,19 @@ eg.
 You can use the --debug command line option in conjunction with the --failfast
 option to have testing stopped at the first failure and the gdb debugger
 automatically launched to debug the failed testcase with a breakpoint set
-in the testStart() fucntion for that testcase.
+in the testStart() function for that testcase.
+
+You can also use the --developer command line option to define the TESTDEV
+preprocessor variable (to turn on developer only test cases, and to have
+all 'hopes' treated as actual 'tests' with pass/fail results).
 
 Writing Tests
 -------------
 
+The test framework may be used for testing Objective-C, ObjectiveC++, C,
+and C++ code.  The test source files must have a .m (for Objective-C and C)
+or a .mm (for Objective-C++ and C++) file extension in order to be recognised.
+
 A minimal test should be a file importing the header "Testing.h"
 (which defines global variables, functions, and standard test macros)
 and containing a main() function implementation which executes the
@@ -196,6 +204,24 @@ You may also arrange to jump to the end of the set if a test fails by wrapping
 the test in a NEED macro.  Doing this also causes the set to be reported as
 Failed if the needed test does not pass.
 
+It's likely that you are writing new tests for a library or framework ...
+and those tests will need to link with that framework.  You should add the
+instructions for that to a GNUmakefile.premable if the directory containing
+your tests, or a GNUmakefile.super in the directory above in th cvase where
+you have multiple test directories.
+eg.
+ADDITIONAL_OBJC_LIBS=-lmyLibrary
+
+When contributing to a test suite, please bracket your new test code using
+#if defined(TESTDEV)
+...
+#endif /* TESTDEV */
+so that it is only built when gnustep-tests is invoked with the --developer
+command line argument.
+This ensures that the new code won't break any existing test code when
+people are simply running the testsuite, and once you are sure that the
+new testcases are correct (and portable to all operating systems), the
+check for TESTDEV can be removed.
 
 Ignoring failed test files
 --------------------------
@@ -232,8 +258,11 @@ tests in the directory are built.
 
 In addition to the preamble/postamble mechanism, the file ../GNUmakefile.super
 is included at the start of the generated makefile (if it exists).  This allows
-all the tests in a suite to use a common makefile fragment to provide
-information for all the testsuite.
+all the test directories in a suite to use a common makefile fragment to provide
+information for the whole testsuite.
+You can also use the GSTESTROOT environment variable to locate resources common
+to the whole testsuite ... it is set automatically by gnustep-test to be the
+absolute path to the topmost directory in the testsuite.
 
 Your system should not make any assumption about the order in which test
 files are built ... the test framework may build many test files in parallel
@@ -257,8 +286,17 @@ if it is invoked without a target, and have a 'clean' target to clean up
 before and after all tests.
 
 
-Ignoring directories
---------------------
+Directory layout
+----------------
+
+A test suite is considered to be a collection of individual test files in
+a single directory or a collection of directories in a hierarchy.
+All directories which contain test files must also contain a TestInfo file
+to mark them as containing files used by the framework, and the root of
+the test suite is considered to be the topmost directory in the hierarchy
+which contains a testInfo file.  The test framework sets the GSTESTROOT
+environment variable to the absolute path of the root of the test suite
+being executed, so scripts and makefiles can use this to locate resources.
 
 The test framework ignores any directory which does not contain a TestInfo
 file.  This feature prevents accidental attempts to treat a project source

TestFramework/Testing.h

@@ -37,6 +37,9 @@
  * are actually unlikely to pass on all systems.
  * The state of this flag is preserved by sets ... on exit from a set
  * it is restored to the state it had on entry.
+ * This flag is ignored if the tests are performed in 'developer' mode
+ * (ie run with the gnustep-tests --developer option and therefore
+ * compiled with the TESTDEV preprocessor macro defined).
  */
 static BOOL testHopeful __attribute__((unused)) = NO;
 
@@ -81,7 +84,8 @@ static NSException *testRaised __attribute__((unused)) = nil;
  *
  * The global variable 'testHopeful' can be set to a non-zero value before
  * calling this function in order to specify that if the condition is
- * not true it should be treated as a dashed hope rather than a failure.
+ * not true it should be treated as a dashed hope rather than a failure
+ * (unless the tests are bing performed in 'developer' mode).
  *
  * If there is a better higher-level test macro available, please use
  * that instead.  In particular, please use the PASS_EQUAL() macro wherever
@@ -107,11 +111,13 @@ static void pass(int passed, const char *format, ...)
       fprintf(stderr, "Passed test:     ");
       testPassed = YES;
     }
+#if	!defined(TESTDEV)
   else if (YES == testHopeful)
     {
       fprintf(stderr, "Dashed hope:     ");
       testPassed = NO;
     }
+#endif
   else
     {
       fprintf(stderr, "Failed test:     ");

TestFramework/gnustep-tests.in

@@ -59,6 +59,9 @@ do
       --debug)
       GSTESTDBG="$GSTESTDIR/gdb.cmds"
       ;;
+      --developer)
+      GSTESTDEV=yes
+      ;;
     --documentation)
       echo
       echo "$0: Script to run the GNUstep testsuite"
@@ -89,6 +92,7 @@ do
       echo "Use 'gnustep-tests --clean' to remove old logs and leftover files."
       echo "Use 'gnustep-tests --failfast' to stop after the first failure."
       echo "Use 'gnustep-tests --debug' to run gdb for any failed tests."
+      echo "Use 'gnustep-tests --developer' to treat hopes as real tests."
       echo "Use 'gnustep-tests --verbose' for full/detailed log output."
       echo "Use 'gnustep-tests --sequential' to disable parallel building."
       echo
@@ -121,20 +125,30 @@ then
   export CC
 fi
 
+GSTESTFLAGS=
 if test "$GSTESTMODE" = "failfast"
+then
+  if test x"$GSTESTDEV" = x"yes"
+  then
+    GSTESTFLAGS="-DTESTDEV=1 -DFAILFAST=1"
+  else
+    GSTESTFLAGS="-DFAILFAST=1"
+  fi
+elif test x"$GSTESTDEV" = x"yes"
+then
+  GSTESTFLAGS="-DTESTDEV=1"
+fi
+
+if test x"$GSTESTFLAGS" != x
 then
   #
-  # We need to add -DFAILFAST=1 to all the code we build as we are wanting
-  # to stop after any test failure.  We do this by using the ADDITIONAL_?FLAGS
+  # We need to add flags to all the code we build.
+  # We do this by using the ADDITIONAL_?FLAGS
   # environment variables supported by gnustep-make.
   #
-  ADDITIONAL_CFLAGS="-DFAILFAST=1 $ADDITIONAL_CFLAGS"
-  export ADDITIONAL_CFLAGS
-  ADDITIONAL_OBJCFLAGS="-DFAILFAST=1 $ADDITIONAL_OBJCFLAGS"
+  ADDITIONAL_OBJCFLAGS="$GSTESTFLAGS $ADDITIONAL_OBJCFLAGS"
   export ADDITIONAL_OBJCFLAGS
-  ADDITIONAL_CCFLAGS="-DFAILFAST=1 $ADDITIONAL_CCFLAGS"
-  export ADDITIONAL_CCFLAGS
-  ADDITIONAL_OBJCCFLAGS="-DFAILFAST=1 $ADDITIONAL_OBJCCFLAGS"
+  ADDITIONAL_OBJCCFLAGS="$GSTESTFLAGS $ADDITIONAL_OBJCCFLAGS"
   export ADDITIONAL_OBJCCFLAGS
 fi
 
@@ -409,6 +423,7 @@ do
   then
     continue
   fi
+
   SUMD=$TESTDIR
   for dir in $SRCDIRS
   do
@@ -417,6 +432,22 @@ do
       continue
     fi
 
+    # Step up through parents of the source directory to find the root of the
+    # test suite (the highest level directory containing a TestInfo file.
+    # Provide that in the environment for use within the makefiles/scripts.
+    GSTESTROOT=$dir
+    parentdir=`dirname $GSTESTROOT`
+    while test -f "$parentdir/TestInfo"
+    do
+      GSTESTROOT="$parentdir"
+      parentdir=`dirname $GSTESTROOT`
+      if test $parentdir = $GSTESTROOT
+      then
+	break
+      fi
+    done
+    export GSTESTROOT
+  
     RUNEXIT=0
     found=yes
     foundany=yes