testing: skip certain tests by listing them in CLIENT_TEST_SKIP

The only way that I found not to execute a test was not to register
it in CPPUnit. FilterTest() does this by replacing a valid test
or test group with a dummy one, SkipTest, which just prints the
test name and that it is skipped.

Registering tests has to be intercepted at multiple levels:
- CPPUNIT_TEST in test suites
- ADD_TEST in ClientTest
- addTest in ClientTest

Not currently intercepted are complete test suites (CPPUNIT_TEST_SUITE).

The main purpose of this patch is to avoid running the time consuming
suspend and interrupt tests, but this feature might also be useful for
other tests, which is why it was implemented in a more general way.

13 files changed, 107 insertions, 17 deletions

diff --git a/HACKING b/HACKING
index fe77fc64..b5ec12b7 100644
--- a/HACKING
+++ b/HACKING
@@ -65,6 +65,10 @@ It understands several environment variables, among them:
 For unattended testing:
 - CLIENT_TEST_FAILURES = comma separated list of tests which are allowed
   to fail without affecting the return code of the test runner
+- CLIENT_TEST_SKIP = comma separated list of tests or test groups which
+  are not to be executed at all; for this to work the test or test group
+  has to be passed through test.h's version of ADD_TEST or FilterTest,
+  which is the case for most tests but not all
 - CLIENT_TEST_LOG = name of server log file, will be copied and reset
   after each sync
 - CLIENT_TEST_ALARM = number of seconds a single test is allowed to run
diff --git a/src/Makefile-gen.am b/src/Makefile-gen.am
index 09d1d8a7..132c5cc2 100644
--- a/src/Makefile-gen.am
+++ b/src/Makefile-gen.am
@@ -84,6 +84,7 @@ libstdc++.a :
 syncevolution_SOURCES = \
 	syncevolution.cpp \
 	$(CORE_SOURCES)
+nodist_syncevolution_SOURCES = ../test/test.cpp
 
 # SYNCEVOLUTION_LDADD will be replaced with libsyncebook.la/libsyncecal.la/libsyncsqlite.la
 # if linking statically against them, empty otherwise;
@@ -139,7 +140,9 @@ syncevo_dbus_server_SOURCES = \
 	syncevo-dbus-server.cpp syncevo-dbus-server.h \
 	DBusSyncClient.cpp DBusSyncClient.h \
 	$(CORE_SOURCES)
-nodist_syncevo_dbus_server_SOURCES = dbus/interfaces/syncevo-marshal.c
+nodist_syncevo_dbus_server_SOURCES = \
+	dbus/interfaces/syncevo-marshal.c \
+	../test/test.cpp
 
 syncevo_dbus_server_LDADD = $(DBUS_GLIB_LIBS) $(KEYRING_LIBS) $(CORE_LDADD)
 syncevo_dbus_server_CPPFLAGS = -DHAVE_CONFIG_H -Idbus/interfaces $(DBUS_GLIB_CFLAGS) $(KEYRING_CFLAGS) $(AM_CPPFLAGS)
@@ -158,6 +161,7 @@ client_test_SOURCES = \
 	../test/ClientTest.h \
 	../test/client-test-main.cpp \
 	$(CORE_SOURCES)
+nodist_client_test_SOURCES = ../test/test.cpp
 
 # Always compile the registry files into client-test because that is
 # the only place where they are compiled with
diff --git a/src/client-test-app.cpp b/src/client-test-app.cpp
index fe3c1f8b..ef7d15de 100644
--- a/src/client-test-app.cpp
+++ b/src/client-test-app.cpp
@@ -22,7 +22,6 @@
 
 #include <ClientTest.h>
 
-#include <cppunit/extensions/HelperMacros.h>
 #include <exception>
 #include <fstream>
 #include <fcntl.h>
diff --git a/src/core/LogRedirect.cpp b/src/core/LogRedirect.cpp
index ae2b16c7..a91d5d5f 100644
--- a/src/core/LogRedirect.cpp
+++ b/src/core/LogRedirect.cpp
@@ -336,7 +336,7 @@ void LogRedirect::process() throw()
 } // namespace SyncEvolution
 
 #ifdef ENABLE_UNIT_TESTS
-#include <cppunit/extensions/HelperMacros.h>
+#include "test.h"
 
 class LogRedirectTest : public CppUnit::TestFixture {
     CPPUNIT_TEST_SUITE(LogRedirectTest);
diff --git a/src/core/Makefile.am b/src/core/Makefile.am
index 706de872..d1ad5c5b 100644
--- a/src/core/Makefile.am
+++ b/src/core/Makefile.am
@@ -1,4 +1,4 @@
-AM_CPPFLAGS = @BACKEND_CPPFLAGS@ @GLIB_CFLAGS@
+AM_CPPFLAGS = @BACKEND_CPPFLAGS@ @GLIB_CFLAGS@ -I$(top_srcdir)/test
 
 # applies to sources in SyncEvolution repository, but not
 # the Funambol C++ client library
diff --git a/src/core/SyncEvolutionCmdline.cpp b/src/core/SyncEvolutionCmdline.cpp
index 7707a4a1..4da5abc4 100644
--- a/src/core/SyncEvolutionCmdline.cpp
+++ b/src/core/SyncEvolutionCmdline.cpp
@@ -778,7 +778,7 @@ void SyncEvolutionCmdline::usage(bool full, const string &error, const string &p
 }
 
 #ifdef ENABLE_UNIT_TESTS
-#include <cppunit/extensions/HelperMacros.h>
+#include "test.h"
 
 /** simple line-by-line diff */
 static string diffStrings(const string &lhs, const string &rhs)
diff --git a/src/core/SyncEvolutionUtil.cpp b/src/core/SyncEvolutionUtil.cpp
index bd2fc3d6..c9e79d64 100644
--- a/src/core/SyncEvolutionUtil.cpp
+++ b/src/core/SyncEvolutionUtil.cpp
@@ -37,7 +37,7 @@
 #include <dirent.h>
 
 #ifdef ENABLE_UNIT_TESTS
-#include <cppunit/extensions/HelperMacros.h>
+#include "test.h"
 CPPUNIT_REGISTRY_ADD_TO_DEFAULT("SyncEvolution");
 #endif
 
diff --git a/src/core/SyncEvolutionUtil.h b/src/core/SyncEvolutionUtil.h
index 31bf2a7e..5c6274bf 100644
--- a/src/core/SyncEvolutionUtil.h
+++ b/src/core/SyncEvolutionUtil.h
@@ -142,7 +142,7 @@ class ReadDir {
  * @verbatim
    #include <config.h>
    #ifdef ENABLE_UNIT_TESTS
-   # include <cppunit/extensions/HelperMacros.h>
+   # include "test.h"
    class Foo : public CppUnit::TestFixture {
        CPPUNIT_TEST_SUITE(foo);
        CPPUNIT_TEST(testBar);
diff --git a/test/ClientTest.cpp b/test/ClientTest.cpp
index bae1b38c..eff9d889 100644
--- a/test/ClientTest.cpp
+++ b/test/ClientTest.cpp
@@ -1691,7 +1691,7 @@ void SyncTests::addTests() {
             ADD_TEST_TO_SUITE(retryTests, SyncTests, testInterruptResumeServerRemove);
             ADD_TEST_TO_SUITE(retryTests, SyncTests, testInterruptResumeServerUpdate);
             ADD_TEST_TO_SUITE(retryTests, SyncTests, testInterruptResumeFull);
-            addTest(retryTests);
+            addTest(FilterTest(retryTests));
         }
 
         if (config.retrySync &&
@@ -1708,7 +1708,7 @@ void SyncTests::addTests() {
             ADD_TEST_TO_SUITE(suspendTests, SyncTests, testUserSuspendServerRemove);
             ADD_TEST_TO_SUITE(suspendTests, SyncTests, testUserSuspendServerUpdate);
             ADD_TEST_TO_SUITE(suspendTests, SyncTests, testUserSuspendFull);
-            addTest(suspendTests);
+            addTest(FilterTest(suspendTests));
         }
 
     }
@@ -3133,10 +3133,10 @@ public:
                 LocalTests *sourcetests =
                     client.createLocalTests(tests->getName() + "::" + config.sourceName, source, config);
                 sourcetests->addTests();
-                tests->addTest(sourcetests);
+                tests->addTest(FilterTest(sourcetests));
             }
         }
-        alltests->addTest(tests);
+        alltests->addTest(FilterTest(tests));
         tests = 0;
 
         // create sync tests with just one source
@@ -3150,7 +3150,7 @@ public:
                 SyncTests *synctests =
                     client.createSyncTests(tests->getName() + "::" + config.sourceName, sources);
                 synctests->addTests();
-                tests->addTest(synctests);
+                tests->addTest(FilterTest(synctests));
             }
         }
 
@@ -3175,7 +3175,7 @@ public:
             SyncTests *synctests =
                 client.createSyncTests(tests->getName() + "::" + name, sources);
             synctests->addTests();
-            tests->addTest(synctests);
+            tests->addTest(FilterTest(synctests));
             synctests = 0;
 
             // now also in reversed order - who knows, it might make a difference
@@ -3183,11 +3183,11 @@ public:
             synctests =
                 client.createSyncTests(tests->getName() + "::" + name_reversed, sources);
             synctests->addTests();
-            tests->addTest(synctests);
+            tests->addTest(FilterTest(synctests));
             synctests = 0;
         }
 
-        alltests->addTest(tests);
+        alltests->addTest(FilterTest(tests));
         tests = 0;
 
         return alltests;
diff --git a/test/ClientTest.h b/test/ClientTest.h
index e59abf94..02070608 100644
--- a/test/ClientTest.h
+++ b/test/ClientTest.h
@@ -37,6 +37,8 @@ typedef EvolutionSyncSource SyncSource;
 #include <SyncML.h>
 #include <TransportAgent.h>
 
+#include "test.h"
+
 #ifdef ENABLE_INTEGRATION_TESTS
 
 #include <cppunit/TestSuite.h>
@@ -1047,8 +1049,7 @@ public:
     ADD_TEST_TO_SUITE(this, _class, _function)
 
 #define ADD_TEST_TO_SUITE(_suite, _class, _function) \
-    _suite->addTest(new CppUnit::TestCaller<_class>(_suite->getName() + "::" #_function, &_class::_function, *this))
-
+    _suite->addTest(FilterTest(new CppUnit::TestCaller<_class>(_suite->getName() + "::" #_function, &_class::_function, *this)))
 
 #endif // ENABLE_INTEGRATION_TESTS
 #endif // INCL_TESTSYNCCLIENT
diff --git a/test/Makefile.am b/test/Makefile.am
index 89549c98..8e429363 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -5,6 +5,7 @@ EXTRA_DIST = \
 	COPYING \
         client-test-main.cpp \
 	test.h \
+	test.cpp \
         Algorithm/Artistic \
         Algorithm/COPYING \
         Algorithm/Diff.pm \
diff --git a/test/test.cpp b/test/test.cpp
new file mode 100644
index 00000000..16c5cfe1
--- /dev/null
+++ b/test/test.cpp
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2009 Intel Corporation
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) version 3.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301  USA
+ */
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#if defined(ENABLE_INTEGRATION_TESTS) || defined(ENABLE_UNIT_TESTS)
+
+#include "test.h"
+
+#include <stdlib.h>
+#include <iostream>
+
+#include <boost/algorithm/string/split.hpp>
+#include <boost/algorithm/string/classification.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+
+class SkipTest : public CppUnit::TestCase {
+public:
+    SkipTest(const std::string &name) :
+        TestCase(name)
+    {}
+    void run (CppUnit::TestResult *result) {
+        std::cerr << getName() << " *** skipped ***\n";
+    }
+};
+
+CppUnit::Test *FilterTest(CppUnit::Test *test)
+{
+    static std::set<std::string> filter;
+    static bool filterValid;
+
+    if (!filterValid) {
+        const char *str = getenv("CLIENT_TEST_SKIP");
+        if (str) {
+            boost::split(filter, str, boost::is_any_of(","));
+        }
+        filterValid = true;
+    }
+
+    std::string name = test->getName();
+    if (filter.find(name) != filter.end()) {
+        delete test;
+        return new SkipTest(name);
+    } else {
+        return test;
+    }
+}
+
+#endif
diff --git a/test/test.h b/test/test.h
index 9704756a..accf4973 100644
--- a/test/test.h
+++ b/test/test.h
@@ -58,6 +58,20 @@ extern const std::string &getCurrentTest();
 // removes special characters like colons and slashes
 extern void simplifyFilename(std::string &filename);
 
+// redefine CPPUNIT_TEST so that we can filter tests
+#undef CPPUNIT_TEST
+#define CPPUNIT_TEST(testMethod) \
+    CPPUNIT_TEST_SUITE_ADD_TEST( \
+        ( FilterTest(new CPPUNIT_NS::TestCaller<TestFixtureType>( \
+                     context.getTestNameFor( #testMethod), \
+                     &TestFixtureType::testMethod, \
+                     context.makeFixture() ) ) ) )
+
+/**
+ * replace test with dummy if filtered out via CLIENT_TEST_SKIP
+ */
+CppUnit::Test *FilterTest(CppUnit::Test *test);
+
 #endif // ENABLE_UNIT_TESTS
 
 /** @endcond */