From ee38ed80d8ae00a5b7e6c80b8022804df52e9265 Mon Sep 17 00:00:00 2001
From: Pekka Paalanen <pekka.paalanen@collabora.com>
Date: Wed, 3 Mar 2021 14:21:01 +0200
Subject: [PATCH] tests: add build option to force skips as failures

This will be useful in CI, where we do not want to see any skips. If
something starts to skip, that's a mistake somewhere, and want to catch
it.

Signed-off-by: Pekka Paalanen <pekka.paalanen@collabora.com>
---
 meson_options.txt          |  6 ++++++
 tests/meson.build          |  1 +
 tests/weston-test-runner.c | 25 ++++++++++++++++++++++++-
 3 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/meson_options.txt b/meson_options.txt
index 239bd2da..ea225a90 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -210,6 +210,12 @@ option(
 	value: true,
 	description: 'Tests: output JUnit XML results'
 )
+option(
+	'test-skip-is-failure',
+	type: 'boolean',
+	value: false,
+	description: 'Tests: consider skip to be a failure'
+)
 option(
 	'test-gl-renderer',
 	type: 'boolean',
diff --git a/tests/meson.build b/tests/meson.build
index 90d776f5..212bba0b 100644
--- a/tests/meson.build
+++ b/tests/meson.build
@@ -276,6 +276,7 @@ test_config_h.set_quoted('WESTON_TEST_REFERENCE_PATH', meson.current_source_dir(
 test_config_h.set_quoted('WESTON_MODULE_MAP', env_modmap)
 test_config_h.set_quoted('WESTON_DATA_DIR', join_paths(meson.current_source_dir(), '..', 'data'))
 test_config_h.set_quoted('TESTSUITE_PLUGIN_PATH', exe_plugin_test.full_path())
+test_config_h.set10('WESTON_TEST_SKIP_IS_FAILURE', get_option('test-skip-is-failure'))
 configure_file(output: 'test-config.h', configuration: test_config_h)
 
 foreach t : tests
diff --git a/tests/weston-test-runner.c b/tests/weston-test-runner.c
index d8becc31..0ab7bc1b 100644
--- a/tests/weston-test-runner.c
+++ b/tests/weston-test-runner.c
@@ -35,6 +35,7 @@
 #include <signal.h>
 #include <getopt.h>
 
+#include "test-config.h"
 #include "weston-test-runner.h"
 #include "weston-testsuite-data.h"
 #include "shared/string-helpers.h"
@@ -243,7 +244,11 @@ result_to_str(enum test_result_code ret)
 		[RESULT_FAIL] = "fail",
 		[RESULT_HARD_ERROR] = "hard error",
 		[RESULT_OK] = "ok",
+#if WESTON_TEST_SKIP_IS_FAILURE
+		[RESULT_SKIP] = "skip error",
+#else
 		[RESULT_SKIP] = "skip",
+#endif
 	};
 
 	assert(ret >= 0 && ret < ARRAY_LENGTH(names));
@@ -284,6 +289,9 @@ run_case(struct wet_testsuite_data *suite_data,
 	case RESULT_SKIP:
 		suite_data->skipped++;
 		skip = " # SKIP";
+#if WESTON_TEST_SKIP_IS_FAILURE
+		fail = "not ";
+#endif
 		break;
 	}
 
@@ -327,10 +335,16 @@ skip_case(struct wet_testsuite_data *suite_data,
 	  const void *test_data,
 	  int iteration)
 {
+	const char *skip_error = "";
 	int iteration_nr = iteration + 1;
 
+#if WESTON_TEST_SKIP_IS_FAILURE
+	skip_error = "not ";
+#endif
+
 	suite_data->counter++;
-	printf("ok %d %s %s/%d # SKIP fixture\n", suite_data->counter,
+	printf("%sok %d %s %s/%d # SKIP fixture\n",
+	       skip_error, suite_data->counter,
 	       suite_data->fixture_name, t->name, iteration_nr);
 }
 
@@ -477,6 +491,11 @@ weston_test_harness_destroy(struct weston_test_harness *harness)
 static enum test_result_code
 counts_to_result(const struct wet_testsuite_data *data)
 {
+#if WESTON_TEST_SKIP_IS_FAILURE
+	if (data->skipped > 0)
+		return RESULT_FAIL;
+#endif
+
 	/* RESULT_SKIP is reserved for fixture setup itself skipping everything */
 	if (data->total == data->passed + data->skipped)
 		return RESULT_OK;
@@ -644,7 +663,11 @@ main(int argc, char *argv[])
 
 		if (ret == RESULT_SKIP) {
 			tap_skip_fixture(&harness->data);
+#if WESTON_TEST_SKIP_IS_FAILURE
+			ret = RESULT_FAIL;
+#else
 			continue;
+#endif
 		}
 
 		if (ret != RESULT_OK && result != RESULT_HARD_ERROR)