From 14a39421804c39f63d8095d5522e662400b6b23f Mon Sep 17 00:00:00 2001 From: Emil Velikov Date: Fri, 10 May 2019 15:47:05 +0100 Subject: [PATCH] ci: fold unreliable tests handing alongside the mustpass ones There is no reason to handle the test separately, at the high-level at least. If needed we can make the results stand out better in the [near] future. Note: this removes the (now dead) --compare-previous. I've considered keeping it, yet since it doesn't attribute the flip-flopping tests it's better to remove it for now. Signed-off-by: Emil Velikov Reviewed-by: Elie Tournier --- ci/.gitlab-ci.yml | 23 ------------------- ci/run_test_suite.sh | 34 ++++++++++------------------ ci/run_tests.sh | 53 +++++++++----------------------------------- 3 files changed, 23 insertions(+), 87 deletions(-) diff --git a/ci/.gitlab-ci.yml b/ci/.gitlab-ci.yml index 661068d..f018500 100644 --- a/ci/.gitlab-ci.yml +++ b/ci/.gitlab-ci.yml @@ -222,26 +222,3 @@ cts gles - gles host: when: always paths: - results/ - -unreliable tests: - stage: test - services: - - docker:dind - before_script: - *docker-setup - script: - - docker run - --ulimit core=99999999999:99999999999 - $RD_CONFIG - -v $PWD:/virglrenderer - $CI_REGISTRY_IMAGE@$DOCKER_IMAGE_DIGEST - bash -c "/virglrenderer/ci/run_tests.sh --deqp-gl-unreliable --deqp-gles-unreliable --piglit-gl-unreliable --piglit-gles-unreliable || true" - - echo "\n\n" - only: - - merge_requests - - schedules - - artifacts: - when: always - paths: - - results/ diff --git a/ci/run_test_suite.sh b/ci/run_test_suite.sh index 1dcec21..d48a266 100755 --- a/ci/run_test_suite.sh +++ b/ci/run_test_suite.sh @@ -8,20 +8,14 @@ BACKENDS="" NUM_THREADS=${NUM_THREADS:-$(expr $(expr $(nproc) / 8) + 1)} COMPARE_BACKENDS=0 -COMPARE_PREVIOUS=0 USE_HOST_GLES=0 TEST_APP="deqp" -VERIFY_UNRELIABLE_TESTS=0 parse_input() { while [ -n "$1" ]; do case $1 in - -u|--unreliable) - VERIFY_UNRELIABLE_TESTS=1 - ;; - -a|--all-backends) BACKENDS="" BACKENDS="$BACKENDS vtest-softpipe" @@ -90,11 +84,6 @@ parse_input() TEST_APP="piglit" ;; - -cp|--compare-previous) - # Compare results against previous runs - COMPARE_PREVIOUS=1 - ;; - *) echo "Unknown flag $1" exit 1 @@ -105,11 +94,6 @@ parse_input() if [[ -z $BACKENDS ]]; then BACKENDS="gpu" fi - - # These two options are incompatible, and one has to be disabled - if [ $VERIFY_UNRELIABLE_TESTS -eq 1 ]; then - COMPARE_PREVIOUS=0 - fi } compare_previous() @@ -158,8 +142,10 @@ interpret_results() { PASSED_TESTS="$1" TOTAL_TESTS="$2" + UNRELIABLE="$3" - if [ $COMPARE_PREVIOUS -eq 1 ]; then + # TODO: Add comparison for the unreliable tests + if [ $UNRELIABLE -eq 0 ]; then compare_previous case $? in 0) @@ -211,12 +197,13 @@ run_test_suite() { local BACKEND="$1" local TEST_NAME="$2" + local UNRELIABLE="$3" # echo "run_test_suite() OUTPUT_PATH: $OUTPUT_PATH" # echo "run_test_suite() LOG_FILE: $LOG_FILE" # echo "run_test_suite() RESULTS_FILE: $RESULTS_FILE" - if [ $VERIFY_UNRELIABLE_TESTS -eq 1 ]; then + if [ $UNRELIABLE -eq 1 ]; then UNRELIABLE_STRING="unreliable " fi @@ -226,7 +213,7 @@ run_test_suite() printf "Running ${UNRELIABLE_STRING}$TEST_APP-$TEST_NAME on $DRIVER_NAME: " fi - if test $VERIFY_UNRELIABLE_TESTS -eq 1; then + if test $UNRELIABLE -eq 1; then TEST_FILE="$IGNORE_TESTS_FILE" if test ! -f $TEST_FILE -o $(wc -l $TEST_FILE | cut -f1 -d' ') -eq 0; then echo "Unreliable: no ignore tests." @@ -239,7 +226,7 @@ run_test_suite() # Don't run GLX tests PIGLIT_TESTS=" -x glx" - if [ $VERIFY_UNRELIABLE_TESTS -eq 1 ]; then + if test $UNRELIABLE -eq 1; then # XXX: Fold the glx exception? PIGLIT_TESTS_CMD="--test-list $TEST_FILE" else @@ -374,12 +361,15 @@ run_test_on_backends() ;; esac - run_test_suite "$BACKEND" "$TEST_NAME" - + # Execute both mustpass and unstable tests + # Only the former twigger an overall run fail + run_test_suite "$BACKEND" "$TEST_NAME" 0 if [ $? -ne 0 ]; then RET=1 fi + run_test_suite "$BACKEND" "$TEST_NAME" 1 + killall -q virgl_test_server done diff --git a/ci/run_tests.sh b/ci/run_tests.sh index ed7d474..16b6a18 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -67,13 +67,8 @@ run_make_check() run_deqp() { - UNRELIABLE="$1" - OGL_BACKEND="$2" - SUITE="$3" - - if [ $UNRELIABLE -eq 1 ]; then - TEST_UNRELIABLE="--unreliable" - fi + OGL_BACKEND="$1" + SUITE="$2" if [ "$SUITE" = "gl" ]; then TEST_SUITE="--gl30 --gl31 --gl32" @@ -90,21 +85,14 @@ run_deqp() ./run_test_suite.sh --deqp ${TEST_SUITE} \ --host-${OGL_BACKEND} \ - ${TEST_UNRELIABLE} \ - ${BACKENDS} \ - --compare-previous + ${BACKENDS} return $? } run_piglit() { - UNRELIABLE="$1" - OGL_BACKEND="$2" - - if [ $UNRELIABLE -eq 1 ]; then - TEST_UNRELIABLE="--unreliable" - fi + OGL_BACKEND="$1" BACKENDS="--backend vtest-softpipe" if [[ -z "$SOFTWARE_ONLY" ]]; then @@ -113,9 +101,7 @@ run_piglit() ./run_test_suite.sh --piglit --gles2 --gles3 \ --host-${OGL_BACKEND} \ - ${TEST_UNRELIABLE} \ - ${BACKENDS} \ - --compare-previous + ${BACKENDS} return $? } @@ -125,51 +111,34 @@ parse_input() RET=0 while [ -n "$1" ]; do echo "" - UNRELIABLE=0 case $1 in --make-check) run_make_check ;; - --deqp-gl-unreliable) - UNRELIABLE=1 - ;& #Fallthrough - --deqp-gl-gl-tests) - run_deqp $UNRELIABLE gl gl + run_deqp gl gl ;; --deqp-gl-gles-tests) - run_deqp $UNRELIABLE gl gles + run_deqp gl gles ;; - --deqp-gles-unreliable) - UNRELIABLE=1 - ;& #Fallthrough - --deqp-gles-gl-tests) - run_deqp $UNRELIABLE gles gl + run_deqp gles gl ;; --deqp-gles-gles-tests) - run_deqp $UNRELIABLE gles gles + run_deqp gles gles ;; - --piglit-gl-unreliable) - UNRELIABLE=1 - ;& #Fallthrough - --piglit-gl) - run_piglit $UNRELIABLE gl + run_piglit gl ;; - --piglit-gles-unreliable) - UNRELIABLE=1 - ;& #Fallthrough - --piglit-gles) - run_piglit $UNRELIABLE gles + run_piglit gles ;; *)