tests: add build option to force skips as failures

This will be useful in CI, where we do not want to see any skips. If
something starts to skip, that's a mistake somewhere, and want to catch
it.

Signed-off-by: Pekka Paalanen <pekka.paalanen@collabora.com>
dev
Pekka Paalanen 4 years ago
parent cde58fd20a
commit ee38ed80d8
  1. 6
      meson_options.txt
  2. 1
      tests/meson.build
  3. 25
      tests/weston-test-runner.c

@ -210,6 +210,12 @@ option(
value: true, value: true,
description: 'Tests: output JUnit XML results' description: 'Tests: output JUnit XML results'
) )
option(
'test-skip-is-failure',
type: 'boolean',
value: false,
description: 'Tests: consider skip to be a failure'
)
option( option(
'test-gl-renderer', 'test-gl-renderer',
type: 'boolean', type: 'boolean',

@ -276,6 +276,7 @@ test_config_h.set_quoted('WESTON_TEST_REFERENCE_PATH', meson.current_source_dir(
test_config_h.set_quoted('WESTON_MODULE_MAP', env_modmap) test_config_h.set_quoted('WESTON_MODULE_MAP', env_modmap)
test_config_h.set_quoted('WESTON_DATA_DIR', join_paths(meson.current_source_dir(), '..', 'data')) test_config_h.set_quoted('WESTON_DATA_DIR', join_paths(meson.current_source_dir(), '..', 'data'))
test_config_h.set_quoted('TESTSUITE_PLUGIN_PATH', exe_plugin_test.full_path()) test_config_h.set_quoted('TESTSUITE_PLUGIN_PATH', exe_plugin_test.full_path())
test_config_h.set10('WESTON_TEST_SKIP_IS_FAILURE', get_option('test-skip-is-failure'))
configure_file(output: 'test-config.h', configuration: test_config_h) configure_file(output: 'test-config.h', configuration: test_config_h)
foreach t : tests foreach t : tests

@ -35,6 +35,7 @@
#include <signal.h> #include <signal.h>
#include <getopt.h> #include <getopt.h>
#include "test-config.h"
#include "weston-test-runner.h" #include "weston-test-runner.h"
#include "weston-testsuite-data.h" #include "weston-testsuite-data.h"
#include "shared/string-helpers.h" #include "shared/string-helpers.h"
@ -243,7 +244,11 @@ result_to_str(enum test_result_code ret)
[RESULT_FAIL] = "fail", [RESULT_FAIL] = "fail",
[RESULT_HARD_ERROR] = "hard error", [RESULT_HARD_ERROR] = "hard error",
[RESULT_OK] = "ok", [RESULT_OK] = "ok",
#if WESTON_TEST_SKIP_IS_FAILURE
[RESULT_SKIP] = "skip error",
#else
[RESULT_SKIP] = "skip", [RESULT_SKIP] = "skip",
#endif
}; };
assert(ret >= 0 && ret < ARRAY_LENGTH(names)); assert(ret >= 0 && ret < ARRAY_LENGTH(names));
@ -284,6 +289,9 @@ run_case(struct wet_testsuite_data *suite_data,
case RESULT_SKIP: case RESULT_SKIP:
suite_data->skipped++; suite_data->skipped++;
skip = " # SKIP"; skip = " # SKIP";
#if WESTON_TEST_SKIP_IS_FAILURE
fail = "not ";
#endif
break; break;
} }
@ -327,10 +335,16 @@ skip_case(struct wet_testsuite_data *suite_data,
const void *test_data, const void *test_data,
int iteration) int iteration)
{ {
const char *skip_error = "";
int iteration_nr = iteration + 1; int iteration_nr = iteration + 1;
#if WESTON_TEST_SKIP_IS_FAILURE
skip_error = "not ";
#endif
suite_data->counter++; suite_data->counter++;
printf("ok %d %s %s/%d # SKIP fixture\n", suite_data->counter, printf("%sok %d %s %s/%d # SKIP fixture\n",
skip_error, suite_data->counter,
suite_data->fixture_name, t->name, iteration_nr); suite_data->fixture_name, t->name, iteration_nr);
} }
@ -477,6 +491,11 @@ weston_test_harness_destroy(struct weston_test_harness *harness)
static enum test_result_code static enum test_result_code
counts_to_result(const struct wet_testsuite_data *data) counts_to_result(const struct wet_testsuite_data *data)
{ {
#if WESTON_TEST_SKIP_IS_FAILURE
if (data->skipped > 0)
return RESULT_FAIL;
#endif
/* RESULT_SKIP is reserved for fixture setup itself skipping everything */ /* RESULT_SKIP is reserved for fixture setup itself skipping everything */
if (data->total == data->passed + data->skipped) if (data->total == data->passed + data->skipped)
return RESULT_OK; return RESULT_OK;
@ -644,7 +663,11 @@ main(int argc, char *argv[])
if (ret == RESULT_SKIP) { if (ret == RESULT_SKIP) {
tap_skip_fixture(&harness->data); tap_skip_fixture(&harness->data);
#if WESTON_TEST_SKIP_IS_FAILURE
ret = RESULT_FAIL;
#else
continue; continue;
#endif
} }
if (ret != RESULT_OK && result != RESULT_HARD_ERROR) if (ret != RESULT_OK && result != RESULT_HARD_ERROR)

Loading…
Cancel
Save