From c22f357464ba64c5a50c07cd04194ebb18d28ab7 Mon Sep 17 00:00:00 2001 From: Pekka Paalanen Date: Mon, 18 Nov 2019 16:13:19 +0200 Subject: [PATCH] doc: overview of the test suite This should lower the barrier to entry for writing more tests. Signed-off-by: Pekka Paalanen --- doc/sphinx/doxygen.ini.in | 6 +- doc/sphinx/index.rst | 1 + doc/sphinx/toc/meson.build | 6 +- doc/sphinx/toc/test-suite-api.rst | 15 ++ doc/sphinx/toc/test-suite.rst | 241 ++++++++++++++++++++++++++++++ 5 files changed, 266 insertions(+), 3 deletions(-) create mode 100644 doc/sphinx/toc/test-suite-api.rst create mode 100644 doc/sphinx/toc/test-suite.rst diff --git a/doc/sphinx/doxygen.ini.in b/doc/sphinx/doxygen.ini.in index 1819cf0d..cfeba090 100644 --- a/doc/sphinx/doxygen.ini.in +++ b/doc/sphinx/doxygen.ini.in @@ -793,7 +793,9 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = @SRC_ROOT@/libweston @SRC_ROOT@/include/libweston +INPUT = @SRC_ROOT@/libweston \ + @SRC_ROOT@/include/libweston \ + @SRC_ROOT@/tests # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses @@ -2083,7 +2085,7 @@ INCLUDE_FILE_PATTERNS = # recursively expanded use the := operator instead of the = operator. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -PREDEFINED = WL_EXPORT= WL_PRINTF(x,y)= +PREDEFINED = WL_EXPORT= WL_PRINTF(x,y)= __attribute__(x)= # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this # tag can be used to specify a list of macro names that should be expanded. The diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst index cd6ca018..d29dc0f8 100644 --- a/doc/sphinx/index.rst +++ b/doc/sphinx/index.rst @@ -6,6 +6,7 @@ Welcome to Weston documentation! :caption: Contents: toc/libweston.rst + toc/test-suite.rst Weston ------ diff --git a/doc/sphinx/toc/meson.build b/doc/sphinx/toc/meson.build index 3cc968bd..6ab3cda5 100644 --- a/doc/sphinx/toc/meson.build +++ b/doc/sphinx/toc/meson.build @@ -1,5 +1,9 @@ # you need to add here any files you add to the toc directory as well -files = [ 'libweston.rst' ] +files = [ + 'libweston.rst', + 'test-suite.rst', + 'test-suite-api.rst', +] foreach file : files configure_file(input: file, output: file, copy: true) diff --git a/doc/sphinx/toc/test-suite-api.rst b/doc/sphinx/toc/test-suite-api.rst new file mode 100644 index 00000000..4f97139f --- /dev/null +++ b/doc/sphinx/toc/test-suite-api.rst @@ -0,0 +1,15 @@ +Reference manual +================ + + +Test harness API +---------------- + +.. doxygengroup:: testharness + :content-only: + +Test harness Internals +---------------------- + +.. doxygengroup:: testharness_private + :content-only: diff --git a/doc/sphinx/toc/test-suite.rst b/doc/sphinx/toc/test-suite.rst new file mode 100644 index 00000000..7fafe3e9 --- /dev/null +++ b/doc/sphinx/toc/test-suite.rst @@ -0,0 +1,241 @@ +Weston test suite +================= + +Weston test suite aims to test features of the Weston compositor and libweston. +The automatic tests are executed as part of ``meson test`` and in the Gitlab CI. +In addition to automatic tests, there are few manual tests that have not been +automated, but being manual means they are also not routinely (or ever) +executed. + + +Test execution +-------------- + +The test execution hierarchy is: + +* ``meson test`` + + * a test program + + * a fixture setup + + * a test + + * a sub-test from a data array + +When ``meson test`` is executed, it will run all defined *test programs* +potentially in parallel and collect their exit status. Therefore it is +important to design each test program to be executable in parallel with every +other test program. + +A **test program** is essentially one ``.c`` source code file that is built into +one executable file (not a library, module, or plugin). Each test program is +possible to run manually without Meson straight from the build directory +without any environment or command line setup, e.g. with GDB or Valgrind. + +A test program may define one **fixture setup** function. The function may be +defined alone or with a data array of an arbitrary data type. If an array is +defined, the fixture setup will be called and all the tests in the program +executed for each element in the array serially. Fixture setups are used for +setting up the Weston compositor for the tests that need it. The array is +useful for running the compositor with different settings for the same tests, +e.g. with Pixman-renderer and GL-renderer. + +**A test** in a test program is defined with one of the macros :c:func:`TEST`, +:c:func:`TEST_P`, or :c:func:`PLUGIN_TEST`. :c:func:`TEST` defines a single +test with no sub-tests. :c:func:`TEST_P` defines a data-driven array of tests: +a set of sub-tests. :c:func:`PLUGIN_TEST` is used specifically by *plugin +tests* that require access to :type:`weston_compositor`. + +All tests and sub-tests are executed serially in a test program. The test +harness does not ``fork()`` which means that any test that crashes or hits an +assert failure will quit the whole test program on the spot, leaving following +tests in that program not executed. + +The test suite has no tests that are expected to fail in general. All tests +that test for a failure must check the exact error condition expected and +succeed if it is met or fail for any other or no error. + + +Types of tests +-------------- + +Aside from manual vs. automatic, there are three types of tests: + +Standalone tests + Standalone tests do not launch the full compositor. + +Plugin tests + Plugin tests launch the Weston compositor and execute the list of tests + from an idle callback handler in the compositor context, blocking the + compositor while they run. + +Client tests + Client tests launch the Weston compositor and execute the list of tests + in a new thread that is created from an idle callback handler. This means + the compositor runs independently from the tests and one can write a test + like as a normal Wayland client. + +The type of all the tests in a test program is defined by the fixture setup +function. A fixture setup function is any defined function with a specific +signature and registered with either :c:func:`DECLARE_FIXTURE_SETUP` or +:c:func:`DECLARE_FIXTURE_SETUP_WITH_ARG`. + + +.. _test-suite-standalone: + +Standalone tests +^^^^^^^^^^^^^^^^ + +Standalone tests do not have a fixture setup function defined in the test +program or the fixture setup function calls +:func:`weston_test_harness_execute_standalone` explicitly. All test cases must +be defined with :c:func:`TEST` or :c:func:`TEST_P`. + +This is the simplest possible test example: + +.. code-block:: c + + TEST(always_success) + { + /* true */ + } + + +.. _test-suite-plugin: + +Plugin tests +^^^^^^^^^^^^ + +Plugin tests must have a fixture setup function that calls +:func:`weston_test_harness_execute_as_plugin`. All test cases must be defined +with :c:func:`PLUGIN_TEST` which declares an implicit function argument +:type:`weston_compositor` ``*compositor``. + +The compositor fixture manufactures the necessary environment variables and the +command line argument array to launch Weston, and calls :func:`wet_main` +directly. An idle task handler is registered, which gets invoked when +initialization is done. All tests are executed from that idle handler, and then +the compositor exits. + +This is an example of a plugin test that just logs a line: + +.. code-block:: c + + static enum test_result_code + fixture_setup(struct weston_test_harness *harness) + { + struct compositor_setup setup; + + compositor_setup_defaults(&setup); + + return weston_test_harness_execute_as_plugin(harness, &setup); + } + DECLARE_FIXTURE_SETUP(fixture_setup); + + PLUGIN_TEST(plugin_registry_test) + { + /* struct weston_compositor *compositor; */ + testlog("Got compositor %p\n", compositor); + } + + +.. _test-suite-client: + +Client tests +^^^^^^^^^^^^ + +Plugin tests must have a fixture setup function that calls +:func:`weston_test_harness_execute_as_client`. All test cases must be +defined with :c:func:`TEST` or :c:func:`TEST_P`. + +The compositor fixture manufactures the necessary environment variables and the +command line argument array to launch Weston, and calls :func:`wet_main` +directly. An idle task handler is registered, which gets invoked when +initialization is done. The idle handler creates a new thread and returns. The +new thread will execute all tests and then signal the compositor to exit. + +This is an incomplete example of an array of sub-tests and another test as +clients: + +.. code-block:: c + + static enum test_result_code + fixture_setup(struct weston_test_harness *harness) + { + struct compositor_setup setup; + + compositor_setup_defaults(&setup); + + return weston_test_harness_execute_as_client(harness, &setup); + } + DECLARE_FIXTURE_SETUP(fixture_setup); + + struct bad_source_rect_args { + int x, y, w, h; + }; + + static const struct bad_source_rect_args bad_source_rect_args[] = { + { -5, 0, 20, 10 }, + { 0, -5, 20, 10 }, + { 5, 6, 0, 10 }, + { 5, 6, 20, 0 }, + { 5, 6, -20, 10 }, + { 5, 6, 20, -10 }, + { -1, -1, 20, 10 }, + { 5, 6, -1, -1 }, + }; + + TEST_P(test_viewporter_bad_source_rect, bad_source_rect_args) + { + const struct bad_source_rect_args *args = data; + struct client *client; + struct wp_viewport *vp; + + client = create_client_and_test_surface(100, 50, 123, 77); + + vp = create_viewport(client); + + testlog("wp_viewport.set_source x=%d, y=%d, w=%d, h=%d\n", + args->x, args->y, args->w, args->h); + set_source(vp, args->x, args->y, args->w, args->h); + + expect_protocol_error(client, &wp_viewport_interface, + WP_VIEWPORT_ERROR_BAD_VALUE); + } + + TEST(test_roundtrip) + { + struct client *client; + + client = create_client_and_test_surface(100, 50, 123, 77); + client_roundtrip(client); + } + + +Writing tests +------------- + +Test programs do not have a ``main()`` of their own. They all share the +``main()`` from the test harness and only define test cases and a fixture +setup. + +It is recommended to have one test program (one ``.c`` file) contain only one +type of tests to keep the fixture setup simple. See +:ref:`test-suite-standalone`, :ref:`test-suite-plugin` and +:ref:`test-suite-client` how to set up each type in a test program. + +.. note:: + + **TODO:** Currently it is not possible to gracefully skip or fail a test. + You can skip with ``exit(RESULT_SKIP)`` but that will quit the whole test + program and all defined tests that were not ran yet will be counted as + failed. You can fail a test by any means, e.g. ``exit(RESULT_FAIL)``, but + the same caveat applies. Succeeded tests must simply return and not call any + exit function. + + +.. toctree:: + :hidden: + + test-suite-api.rst