ci: fold unreliable tests handing alongside the mustpass ones

There is no reason to handle the test separately, at the high-level at
least.

If needed we can make the results stand out better in the [near] future.

Note: this removes the (now dead) --compare-previous. I've considered
keeping it, yet since it doesn't attribute the flip-flopping tests it's
better to remove it for now.

Signed-off-by: Emil Velikov <emil.velikov@collabora.com>
Reviewed-by: Elie Tournier <elie.tournier@collabora.com>

ci/.gitlab-ci.yml

@@ -222,26 +222,3 @@ cts gles - gles host:
     when: always
     paths:
       - results/
-
-unreliable tests:
-  stage: test
-  services:
-    - docker:dind
-  before_script:
-    *docker-setup
-  script:
-    - docker run
-             --ulimit core=99999999999:99999999999
-             $RD_CONFIG
-             -v $PWD:/virglrenderer
-             $CI_REGISTRY_IMAGE@$DOCKER_IMAGE_DIGEST
-             bash -c "/virglrenderer/ci/run_tests.sh --deqp-gl-unreliable --deqp-gles-unreliable --piglit-gl-unreliable --piglit-gles-unreliable || true"
-    - echo "\n\n"
-  only:
-    - merge_requests
-    - schedules
-
-  artifacts:
-    when: always
-    paths:
-      - results/

ci/run_test_suite.sh

@@ -8,20 +8,14 @@ BACKENDS=""
 NUM_THREADS=${NUM_THREADS:-$(expr $(expr $(nproc) / 8) + 1)}
 
 COMPARE_BACKENDS=0
-COMPARE_PREVIOUS=0
 USE_HOST_GLES=0
 TEST_APP="deqp"
-VERIFY_UNRELIABLE_TESTS=0
 
 parse_input()
 {
    while  [ -n "$1" ]; do
       case $1 in
 
-      -u|--unreliable)
-         VERIFY_UNRELIABLE_TESTS=1
-         ;;
-
       -a|--all-backends)
          BACKENDS=""
          BACKENDS="$BACKENDS vtest-softpipe"
@@ -90,11 +84,6 @@ parse_input()
          TEST_APP="piglit"
          ;;
 
-      -cp|--compare-previous)
-         # Compare results against previous runs
-         COMPARE_PREVIOUS=1
-         ;;
-
    	*)
    	   echo "Unknown flag $1"
    	   exit 1
@@ -105,11 +94,6 @@ parse_input()
    if [[ -z $BACKENDS ]]; then
       BACKENDS="gpu"
    fi
-
-   # These two options are incompatible, and one has to be disabled
-   if [ $VERIFY_UNRELIABLE_TESTS -eq 1 ]; then
-      COMPARE_PREVIOUS=0
-   fi
 }
 
 compare_previous()
@@ -158,8 +142,10 @@ interpret_results()
 {
    PASSED_TESTS="$1"
    TOTAL_TESTS="$2"
+   UNRELIABLE="$3"
 
-   if [ $COMPARE_PREVIOUS -eq 1 ]; then
+   # TODO: Add comparison for the unreliable tests
+   if [ $UNRELIABLE -eq 0 ]; then
       compare_previous
       case $? in
          0)
@@ -211,12 +197,13 @@ run_test_suite()
 {
    local BACKEND="$1"
    local TEST_NAME="$2"
+   local UNRELIABLE="$3"
 
 #   echo "run_test_suite() OUTPUT_PATH: $OUTPUT_PATH"
 #   echo "run_test_suite() LOG_FILE: $LOG_FILE"
 #   echo "run_test_suite() RESULTS_FILE: $RESULTS_FILE"
 
-   if [ $VERIFY_UNRELIABLE_TESTS -eq 1 ]; then
+   if [ $UNRELIABLE -eq 1 ]; then
       UNRELIABLE_STRING="unreliable "
    fi
 
@@ -226,7 +213,7 @@ run_test_suite()
       printf "Running ${UNRELIABLE_STRING}$TEST_APP-$TEST_NAME on $DRIVER_NAME: "
    fi
 
-   if test $VERIFY_UNRELIABLE_TESTS -eq 1; then
+   if test $UNRELIABLE -eq 1; then
       TEST_FILE="$IGNORE_TESTS_FILE"
       if test ! -f $TEST_FILE -o $(wc -l $TEST_FILE | cut -f1 -d' ') -eq 0; then
          echo "Unreliable: no ignore tests."
@@ -239,7 +226,7 @@ run_test_suite()
       # Don't run GLX tests
       PIGLIT_TESTS=" -x glx"
 
-      if [ $VERIFY_UNRELIABLE_TESTS -eq 1 ]; then
+      if test $UNRELIABLE -eq 1; then
          # XXX: Fold the glx exception?
          PIGLIT_TESTS_CMD="--test-list $TEST_FILE"
       else
@@ -374,12 +361,15 @@ run_test_on_backends()
                ;;
          esac
 
-         run_test_suite "$BACKEND" "$TEST_NAME"
+         # Execute both mustpass and unstable tests
-
+         # Only the former twigger an overall run fail
+         run_test_suite "$BACKEND" "$TEST_NAME" 0
          if [ $? -ne 0 ]; then
             RET=1
          fi
 
+         run_test_suite "$BACKEND" "$TEST_NAME" 1
+
          killall -q virgl_test_server
    done
 

ci/run_tests.sh

@@ -67,13 +67,8 @@ run_make_check()
 
 run_deqp()
 {
-   UNRELIABLE="$1"
+   OGL_BACKEND="$1"
-   OGL_BACKEND="$2"
+   SUITE="$2"
-   SUITE="$3"
-
-   if [ $UNRELIABLE -eq 1 ]; then
-      TEST_UNRELIABLE="--unreliable"
-   fi
 
    if [ "$SUITE" = "gl" ]; then
       TEST_SUITE="--gl30 --gl31 --gl32"
@@ -90,21 +85,14 @@ run_deqp()
 
    ./run_test_suite.sh --deqp ${TEST_SUITE} \
       --host-${OGL_BACKEND} \
-      ${TEST_UNRELIABLE} \
+      ${BACKENDS}
-      ${BACKENDS} \
-      --compare-previous
 
    return $?
 }
 
 run_piglit()
 {
-   UNRELIABLE="$1"
+   OGL_BACKEND="$1"
-   OGL_BACKEND="$2"
-
-   if [ $UNRELIABLE -eq 1 ]; then
-      TEST_UNRELIABLE="--unreliable"
-   fi
 
    BACKENDS="--backend vtest-softpipe"
    if [[ -z "$SOFTWARE_ONLY" ]]; then
@@ -113,9 +101,7 @@ run_piglit()
 
    ./run_test_suite.sh --piglit --gles2 --gles3 \
       --host-${OGL_BACKEND} \
-      ${TEST_UNRELIABLE} \
+      ${BACKENDS}
-      ${BACKENDS} \
-      --compare-previous
 
    return $?
 }
@@ -125,51 +111,34 @@ parse_input()
    RET=0
    while  [ -n "$1" ]; do
       echo ""
-      UNRELIABLE=0
 
       case $1 in
          --make-check)
          run_make_check
          ;;
 
-         --deqp-gl-unreliable)
-         UNRELIABLE=1
-         ;& #Fallthrough
-
          --deqp-gl-gl-tests)
-         run_deqp $UNRELIABLE gl gl
+         run_deqp gl gl
          ;;
 
          --deqp-gl-gles-tests)
-         run_deqp $UNRELIABLE gl gles
+         run_deqp gl gles
          ;;
 
-         --deqp-gles-unreliable)
-         UNRELIABLE=1
-         ;& #Fallthrough
-
          --deqp-gles-gl-tests)
-         run_deqp $UNRELIABLE gles gl
+         run_deqp gles gl
          ;;
 
          --deqp-gles-gles-tests)
-         run_deqp $UNRELIABLE gles gles
+         run_deqp gles gles
          ;;
 
-         --piglit-gl-unreliable)
-         UNRELIABLE=1
-         ;& #Fallthrough
-
          --piglit-gl)
-         run_piglit $UNRELIABLE gl
+         run_piglit gl
          ;;
 
-         --piglit-gles-unreliable)
-         UNRELIABLE=1
-         ;& #Fallthrough
-
          --piglit-gles)
-         run_piglit $UNRELIABLE gles
+         run_piglit gles
          ;;
 
          *)