ci: Switch to new test-runner

Reviewed-By: Gert Wollny <gert.wollny@collabora.com>
Signed-off-by: Robert Foss <robert.foss@collabora.com>
Signed-off-by: Jakob Bornecrantz <jakob@collabora.com>

ci/.gitlab-ci.yml

@@ -13,6 +13,7 @@ variables:
 
 stages:
   - build
+  - sanity test
   - test
 
 build docker image:
@@ -39,7 +40,107 @@ build docker image:
     paths:
       - results/
 
-test:
+make check:
+  stage: sanity test
+  services:
+    - docker:dind
+  before_script:
+    - mkdir -p ccache
+    - mkdir -p results
+  script:
+    - echo core > /proc/sys/kernel/core_pattern || true
+    - echo 0 > /proc/sys/kernel/core_uses_pid || true
+    - if [[ -n "$RENDER_DEVICE" ]]; then RD_CONFIG="--device=$RENDER_DEVICE -e RENDER_DEVICE=$RENDER_DEVICE"; fi
+    - echo RD_CONFIG=$RD_CONFIG
+    - time docker run
+             --ulimit core=99999999999:99999999999
+             $RD_CONFIG
+             -v $PWD:/virglrenderer
+             $CI_REGISTRY_IMAGE:latest
+             bash -c "/virglrenderer/ci/run-tests.sh --make-check"
+             2>&1 | tee results/docker_test_log.txt
+
+  artifacts:
+    when: always
+    paths:
+      - results/
+
+piglit:
+  stage: test
+  services:
+    - docker:dind
+  before_script:
+    - mkdir -p ccache
+    - mkdir -p results
+  script:
+    - echo core > /proc/sys/kernel/core_pattern || true
+    - echo 0 > /proc/sys/kernel/core_uses_pid || true
+    - if [[ -n "$RENDER_DEVICE" ]]; then RD_CONFIG="--device=$RENDER_DEVICE -e RENDER_DEVICE=$RENDER_DEVICE"; fi
+    - echo RD_CONFIG=$RD_CONFIG
+    - time docker run
+             --ulimit core=99999999999:99999999999
+             $RD_CONFIG
+             -v $PWD:/virglrenderer
+             $CI_REGISTRY_IMAGE:latest
+             bash -c "/virglrenderer/ci/run-tests.sh --piglit-gl --piglit-gles"
+             2>&1 | tee results/docker_test_log.txt
+
+  artifacts:
+    when: always
+    paths:
+      - results/
+
+cts - gl host:
+  stage: test
+  services:
+    - docker:dind
+  before_script:
+    - mkdir -p ccache
+    - mkdir -p results
+  script:
+    - echo core > /proc/sys/kernel/core_pattern || true
+    - echo 0 > /proc/sys/kernel/core_uses_pid || true
+    - if [[ -n "$RENDER_DEVICE" ]]; then RD_CONFIG="--device=$RENDER_DEVICE -e RENDER_DEVICE=$RENDER_DEVICE"; fi
+    - echo RD_CONFIG=$RD_CONFIG
+    - time docker run
+             --ulimit core=99999999999:99999999999
+             $RD_CONFIG
+             -v $PWD:/virglrenderer
+             $CI_REGISTRY_IMAGE:latest
+             bash -c "/virglrenderer/ci/run-tests.sh --deqp-gl"
+             2>&1 | tee results/docker_test_log.txt
+
+  artifacts:
+    when: always
+    paths:
+      - results/
+
+cts - gles host:
+  stage: test
+  services:
+    - docker:dind
+  before_script:
+    - mkdir -p ccache
+    - mkdir -p results
+  script:
+    - echo core > /proc/sys/kernel/core_pattern || true
+    - echo 0 > /proc/sys/kernel/core_uses_pid || true
+    - if [[ -n "$RENDER_DEVICE" ]]; then RD_CONFIG="--device=$RENDER_DEVICE -e RENDER_DEVICE=$RENDER_DEVICE"; fi
+    - echo RD_CONFIG=$RD_CONFIG
+    - time docker run
+             --ulimit core=99999999999:99999999999
+             $RD_CONFIG
+             -v $PWD:/virglrenderer
+             $CI_REGISTRY_IMAGE:latest
+             bash -c "/virglrenderer/ci/run-tests.sh --deqp-gles"
+             2>&1 | tee results/docker_test_log.txt
+
+  artifacts:
+    when: always
+    paths:
+      - results/
+
+unreliable tests:
   stage: test
   services:
     - docker:dind
@@ -49,14 +150,14 @@ test:
   script:
     - echo core > /proc/sys/kernel/core_pattern || true
     - echo 0 > /proc/sys/kernel/core_uses_pid || true
-    - if test -e $RENDER_DEVICE; then RD_CONFIG="--device=$RENDER_DEVICE -e RENDER_DEVICE=$RENDER_DEVICE"; fi
+    - if [[ -n "$RENDER_DEVICE" ]]; then RD_CONFIG="--device=$RENDER_DEVICE -e RENDER_DEVICE=$RENDER_DEVICE"; fi
     - echo RD_CONFIG=$RD_CONFIG
     - time docker run
              --ulimit core=99999999999:99999999999
              $RD_CONFIG
              -v $PWD:/virglrenderer
              $CI_REGISTRY_IMAGE:latest
-             /virglrenderer/ci/run-tests.sh
+             bash -c "/virglrenderer/ci/run-tests.sh --deqp-gl-unreliable --deqp-gles-unreliable --piglit-gl-unreliable --piglit-gles-unreliable || true"
              2>&1 | tee results/docker_test_log.txt
 
   artifacts:

ci/Dockerfile

@@ -114,9 +114,7 @@ RUN git clone --shallow-since="$GIT_DATE" https://github.com/KhronosGroup/VK-GL-
     cmake -DDEQP_TARGET=x11_egl -DCMAKE_BUILD_TYPE=Release .. && \
     make -j$(nproc) && \
     find . -name CMakeFiles | xargs rm -rf && \
-    find . -name lib\*.a | xargs rm -rf && \
+    find . -name lib\*.a | xargs rm -rf
-    mv /VK-GL-CTS/build /usr/local/VK-GL-CTS && \
-    rm -rf /VK-GL-CTS
 WORKDIR /
 
 ARG KNOWN_GOOD_PIGLIT=1a2f49f17fb45

ci/run-deqp.sh

@@ -1,221 +0,0 @@
-#!/bin/bash
-
-set -x
-
-ONLY_GLES2=no
-
-for arg
-do
-    case "$arg" in
-        --with-vtest)
-            WITH_VTEST=1
-            ;;
-        --host-gl)
-            HOST_GL=1
-            ;;
-        --only-softpipe)
-            ONLY_SOFTPIPE_PASS=yes
-            ;;
-        *)
-            echo "Unknown argument"
-            exit 1
-            ;;
-     esac
-done
-
-export PATH=$PATH:/usr/local/go/bin
-export LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib:/usr/local/lib/x86_64-linux-gnu
-#export MESA_GLES_VERSION_OVERRIDE=3.1
-
-if [[ -n $WITH_VTEST ]]; then
-   nohup Xvfb :0 -screen 0 1024x768x24 &
-else
-   startx &
-fi
-sleep 3
-export DISPLAY=:0
-
-if [[ -n "$LIBGL_ALWAYS_SOFTWARE" ]]; then
-    HOST_GALLIUM_DRIVER="_${GALLIUM_DRIVER}"
-fi
-
-if [[ -n "$HOST_GL" ]]; then
-    echo "Running tests with GL on the host"
-    RESULTS_DIR=/virglrenderer/results/gl_host${HOST_GALLIUM_DRIVER}
-    PREVIOUS_RESULTS_DIR=/virglrenderer/ci/previous_results/gl_host${HOST_GALLIUM_DRIVER}
-else
-    echo "Running tests with GLES on the host"
-    RESULTS_DIR=/virglrenderer/results/es_host${HOST_GALLIUM_DRIVER}
-    PREVIOUS_RESULTS_DIR=/virglrenderer/ci/previous_results/es_host${HOST_GALLIUM_DRIVER}
-fi
-mkdir -p $RESULTS_DIR
-
-if [[ -n "$WITH_VTEST" ]]; then
-    if [[ -n "$HOST_GL" ]]; then
-        if [[ -n "$ONLY_SOFTPIPE_PASS" ]]; then
-            # Softpipe only supports GL 3.3, but for guest GLES 3.1
-            # one GL 4.4 property is needed
-            VTEST_USE_EGL_SURFACELESS=1 nohup /virglrenderer/vtest/virgl_test_server >$RESULTS_DIR/vtest_gl.log 2>&1  &
-        else
-            VTEST_USE_EGL_SURFACELESS=1 nohup /virglrenderer/vtest/virgl_test_server >$RESULTS_DIR/vtest_gl.log  2>&1 &
-        fi
-   else
-       VTEST_USE_EGL_SURFACELESS=1 VTEST_USE_GLES=1 nohup /virglrenderer/vtest/virgl_test_server >$RESULTS_DIR/vtest_gles.log 2>&1  &
-   fi
-
-   sleep 1
-
-   # Don't set the GALLIUM_DRIVER to virpipe before
-   # initializing virgl_test_server
-   export GALLIUM_DRIVER=virpipe
-fi
-
-if [[ "x$ONLY_SOFTPIPE_PASS"="xyes" ]] ; then
-    time deqp --threads=$NUM_THREADS \
-     --cts-build-dir=/usr/local/VK-GL-CTS/ \
-     --test-names-file=/virglrenderer/ci/softpipe-gles-pass.txt \
-     --print-failing \
-     --results-file=$RESULTS_DIR/deqp_results.txt
-else
-    time deqp --threads=$NUM_THREADS \
-     --cts-build-dir=/usr/local/VK-GL-CTS/ \
-     --test-names-file=/virglrenderer/ci/deqp-gles2-list.txt \
-     --test-names-file=/virglrenderer/ci/deqp-gles3-list.txt \
-     --test-names-file=/virglrenderer/ci/deqp-gles31-list.txt \
-     --print-failing \
-     --results-file=$RESULTS_DIR/deqp_results.txt
-fi
-cp -rf /tmp/dEQP/* $RESULTS_DIR/.
-
-# Remove header
-sed -i "/#/d" $RESULTS_DIR/deqp_results.txt
-
-# TODO: These tests are not reliable when run on radeonsi, someone should fix them and then remove these lines
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES3.functional.fbo.msaa.2_samples.rgb8"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES3.functional.fbo.msaa.4_samples.rgb8"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES3.functional.fbo.msaa.8_samples.rgb8"
-
-# TODO: These tests aren't reliable either
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.shaders.sample_variables.sample_pos.correctness.singlesample_texture"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.shaders.sample_variables.sample_pos.correctness.multisample_texture_1"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.shaders.sample_variables.sample_pos.correctness.multisample_texture_2"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.shaders.sample_variables.sample_pos.correctness.multisample_texture_4"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.shaders.sample_variables.sample_pos.correctness.multisample_texture_8"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.shaders.sample_variables.sample_pos.correctness.singlesample_rbo"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.shaders.sample_variables.sample_pos.correctness.multisample_rbo_1"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.shaders.sample_variables.sample_pos.correctness.multisample_rbo_2"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.shaders.sample_variables.sample_pos.correctness.multisample_rbo_4"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.shaders.sample_variables.sample_pos.correctness.multisample_rbo_8"
-
-# TODO: These tests aren't reliable either
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES3.functional.negative_api.shader.sampler_parameterfv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES3.functional.negative_api.shader.sampler_parameterf"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES3.functional.negative_api.shader.sampler_parameteriv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES3.functional.negative_api.shader.sampler_parameteri"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES3.functional.negative_api.state.get_synciv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.callbacks.shader.sampler_parameterfv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.callbacks.shader.sampler_parameterf"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.callbacks.shader.sampler_parameteriv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.callbacks.shader.sampler_parameteri"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.callbacks.shader.srgb_decode_samplerparameterfv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.callbacks.shader.srgb_decode_samplerparameterf"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.callbacks.shader.srgb_decode_samplerparameteriv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.callbacks.shader.srgb_decode_samplerparameteri"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.callbacks.state.get_synciv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.get_error.compute.exceed_atomic_counters_limit"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.get_error.shader.sampler_parameterfv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.get_error.shader.sampler_parameterf"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.get_error.shader.sampler_parameteriv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.get_error.shader.sampler_parameteri"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.get_error.shader.srgb_decode_samplerparameterfv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.get_error.shader.srgb_decode_samplerparameterf"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.get_error.shader.srgb_decode_samplerparameteriv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.get_error.shader.srgb_decode_samplerparameteri"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.log.shader.sampler_parameterfv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.log.shader.sampler_parameterf"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.log.shader.sampler_parameteriv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.log.shader.sampler_parameteri"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.log.shader.srgb_decode_samplerparameterfv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.log.shader.srgb_decode_samplerparameterf"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.log.shader.srgb_decode_samplerparameteriv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.log.shader.srgb_decode_samplerparameteri"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.get_error.state.get_synciv"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.debug.negative_coverage.log.state.get_synciv"
-
-
-# TODO: These tests fail on softpipe, someone should fix them and then remove these lines
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.shaders.builtin_functions.uniform.findLSBMinusOne.lowp_vertex"
-FLIP_FLOPS="$FLIP_FLOPS dEQP-GLES31.functional.shaders.builtin_functions.uniform.findLSBMinusOne.highp_fragment"
-
-for TEST_NAME in $FLIP_FLOPS; do
-    sed -i "\:$TEST_NAME:d" $RESULTS_DIR/deqp_results.txt $PREVIOUS_RESULTS_DIR/deqp_results.txt
-done
-
-# Sort results files
-sort -V $RESULTS_DIR/deqp_results.txt -o $RESULTS_DIR/deqp_results.txt
-sort -V $PREVIOUS_RESULTS_DIR/deqp_results.txt -o $PREVIOUS_RESULTS_DIR/deqp_results.txt
-
-# These warnings add too much variability
-sed -i "s/QualityWarning/Pass/g" $RESULTS_DIR/deqp_results.txt $PREVIOUS_RESULTS_DIR/deqp_results.txt
-sed -i "s/CompatibilityWarning/Pass/g" $RESULTS_DIR/deqp_results.txt $PREVIOUS_RESULTS_DIR/deqp_results.txt
-
-diff -u $PREVIOUS_RESULTS_DIR/deqp_results.txt $RESULTS_DIR/deqp_results.txt 2>&1 > $RESULTS_DIR/deqp_diff.txt
-if [ $? -ne 0 ]; then
-   cat $RESULTS_DIR/deqp_diff.txt
-   touch /virglrenderer/results/regressions_detected
-else
-   rm $RESULTS_DIR/deqp_diff.txt
-fi
-
-
-if [[ "x$ONLY_SOFTPIPE_PASS" != "xyes" ]] ; then
-    mkdir -p $RESULTS_DIR/piglit
-
-    PIGLIT_TESTS="-x glx"
-    if [[ -z "$HOST_GL" ]]; then
-        PIGLIT_TESTS="$PIGLIT_TESTS -t gles2 -t gles3"
-    fi
-
-    # Hits this assertion on i965:
-    # compiler/brw_fs_visitor.cpp:444: void fs_visitor::emit_fb_writes(): Assertion `!prog_data->dual_src_blend || key->nr_color_regions == 1` failed
-    PIGLIT_TESTS="$PIGLIT_TESTS -x arb_blend_func_extended-fbo-extended-blend-pattern_gles2"
-
-    time piglit run --platform x11_egl \
-         --jobs $NUM_JOBS \
-         $PIGLIT_TESTS \
-         gpu \
-         $RESULTS_DIR/piglit
-
-    piglit summary console $RESULTS_DIR/piglit | head -n -17 > $RESULTS_DIR/piglit/results.txt
-
-    # TODO: These tests are not reliable when run on radeonsi, someone should fix them and then remove these lines
-    FLIP_FLOPS=""
-    FLIP_FLOPS="$FLIP_FLOPS spec/arb_framebuffer_srgb/blit renderbuffer srgb_to_linear downsample enabled clear"
-    FLIP_FLOPS="$FLIP_FLOPS spec/arb_framebuffer_srgb/blit texture srgb_to_linear msaa enabled clear"
-    FLIP_FLOPS="$FLIP_FLOPS spec/arb_shader_image_load_store/shader-mem-barrier/fragment shader/'volatile' qualifier memory barrier test/modulus="
-    FLIP_FLOPS="$FLIP_FLOPS spec/arb_shader_image_load_store/shader-mem-barrier/fragment shader/'coherent' qualifier memory barrier test/modulus="
-    FLIP_FLOPS="$FLIP_FLOPS spec/arb_shader_image_load_store/shader-mem-barrier/tessellation control shader/'volatile' qualifier memory barrier test/modulus="
-    FLIP_FLOPS="$FLIP_FLOPS spec/arb_shader_image_load_store/shader-mem-barrier/tessellation control shader/'coherent' qualifier memory barrier test/modulus="
-    FLIP_FLOPS="$FLIP_FLOPS spec/ext_transform_instanced/draw-auto instanced"
-    for TEST_NAME in $FLIP_FLOPS; do
-        sed -i "\:$TEST_NAME:d" $RESULTS_DIR/piglit_results $PREVIOUS_RESULTS_DIR/piglit_results.txt
-    done
-
-    # This distinction adds too much variability
-    sed -i "s/crash/fail/g" $RESULTS_DIR/piglit_results $PREVIOUS_RESULTS_DIR/piglit_results.txt
-
-    # Sort results files
-    sort -V $RESULTS_DIR/piglit_results -o $RESULTS_DIR/piglit_results
-    sort -V $PREVIOUS_RESULTS_DIR/piglit_results.txt -o $PREVIOUS_RESULTS_DIR/piglit_results.txt
-
-    diff -u $PREVIOUS_RESULTS_DIR/piglit_results.txt $RESULTS_DIR/piglit/results.txt 2>&1 > $RESULTS_DIR/piglit_diff.txt
-    if [ $? -ne 0 ]; then
-        cat $RESULTS_DIR/piglit_diff.txt
-        touch /virglrenderer/results/regressions_detected
-    else
-        rm $RESULTS_DIR/piglit_diff.txt
-    fi
-
-fi
-
-killall virgl_test_server

ci/run-tests.sh

@@ -1,36 +1,34 @@
 #!/bin/bash
 
-set -x
+run_setup()
+{
+   set -x
 
-#DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends ninja-build meson
+   # Let .gitlab-ci or local ci runner set
+   # desired thread count
+   NUM_THREADS=${NUM_THREADS:-"$(nproc)"}
+   export NUM_THREADS
+   echo "Using $NUM_THREADS threads"
 
-# Let .gitlab-ci or local ci runner set
+   export CCACHE_BASEDIR=/virglrenderer
-# desired thread count
+   export CCACHE_DIR=/virglrenderer/ccache
-NUM_THREADS=${NUM_THREADS:-"$(nproc)"}
+   export PATH="/usr/lib/ccache:$PATH"
-export NUM_THREADS
+   mkdir -p $CCACHE_DIR
-echo "Using $NUM_THREADS threads"
+   ccache -s
 
-# To prevent hitting assertions such as the below:
+   # To prevent hitting assertions such as the below:
-# sb/sb_sched.cpp:1207:schedule_alu: Assertion `!"unscheduled pending instructions"' failed.
+   # sb/sb_sched.cpp:1207:schedule_alu: Assertion `!"unscheduled pending instructions"' failed.
-export R600_DEBUG=nosb
+   export R600_DEBUG=nosb
-
-export CCACHE_BASEDIR=/virglrenderer
-export CCACHE_DIR=/virglrenderer/ccache
-export PATH="/usr/lib/ccache:$PATH"
-mkdir -p $CCACHE_DIR
-ccache -s
 
+   # If render node, like /dev/dri/renderD128, hasn't been set
+   # or exists use softpipe instead of HW GPU.
+   if [[ ! -c $RENDER_DEVICE ]]; then
+      export SOFTWARE_ONLY=1
+   fi
 
-# If render node, like /dev/dri/renderD128, hasn't been set
+   set +x
-# or exists use softpipe instead of HW GPU.
-if [[ ! -c $RENDER_DEVICE ]]; then
-    export LIBGL_ALWAYS_SOFTWARE=1
-    export GALLIVM_PERF=no_filter_hacks
-    export GALLIUM_DRIVER=softpipe
-    LIMIT_TESTSET=--only-softpipe
-fi
 
-if [[ $LOCAL_MESA ]]; then
+   if [[ $LOCAL_MESA ]]; then
       cd $LOCAL_MESA && \
       mkdir -p build  && \
       meson build/ && \
@@ -39,31 +37,180 @@ if [[ $LOCAL_MESA ]]; then
       if [ $? -ne 0 ]; then
         exit 1
       fi
-fi
+   fi
 
+   VIRGL_PATH="/virglrenderer"
+   rm -rf $VIRGL_PATH/results/
+   mkdir -p $VIRGL_PATH/results/
 
-rm -rf /virglrenderer/results/
+   if [ -d "$VIRGL_PATH" ]; then
-mkdir -p /virglrenderer/results/
+       cd $VIRGL_PATH
-cd /virglrenderer
+       ./autogen.sh --prefix=/usr/local --enable-debug --enable-tests
-./autogen.sh --prefix=/usr/local --enable-debug --enable-tests
+       make -j$NUM_THREADS install
-VRENDTEST_USE_EGL_SURFACELESS=1 make check
+   fi
-if [ $? -ne 0 ]; then
-    touch /virglrenderer/results/regressions_detected
-fi
-mkdir -p /virglrenderer/results/make_check
-cp tests/test*.log /virglrenderer/results/make_check/
-make -j$NUM_THREADS install
 
+   CI_DIR=$(dirname $(readlink -f "$0"))
+   cd $CI_DIR
+}
 
-# Stop testing process if a failure have been found
+run_make_check()
-if [ -f /virglrenderer/results/regressions_detected ]; then
+{
-   exit 1
+   (
-fi
+      cd /virglrenderer
+      mkdir -p /virglrenderer/results/make_check
+      VRENDTEST_USE_EGL_SURFACELESS=1 make -j$NUM_THREADS check --no-print-directory
+      RET=$?
+      cp tests/test*.log /virglrenderer/results/make_check/
+      return $RET
+   )
+}
+
+run_deqp_gl()
+{
+   UNRELIABLE="$1"
+
+   if [ $UNRELIABLE -eq 1 ]; then
+      TEST_UNRELIABLE="--unreliable"
+   fi
+
+   BACKENDS="--backend vtest-softpipe"
+   if [[ -z "$SOFTWARE_ONLY" ]]; then
+      BACKENDS="${BACKENDS} --backend vtest-gpu"
+   fi
+
+   ./run_test_suite.sh --deqp --gles2 --gles3 --gles31 \
+      --host-gl \
+      ${TEST_UNRELIABLE} \
+      ${BACKENDS} \
+      --compare-previous
+
+   return $?
+}
+
+run_deqp_gles()
+{
+   UNRELIABLE="$1"
+
+   if [ $UNRELIABLE -eq 1 ]; then
+      TEST_UNRELIABLE="--unreliable"
+   fi
+
+   BACKENDS="--backend vtest-softpipe"
+   if [[ -z "$SOFTWARE_ONLY" ]]; then
+      BACKENDS="${BACKENDS} --backend vtest-gpu"
+   fi
+
+   ./run_test_suite.sh --deqp --gles2 --gles3 --gles31 \
+      --host-gles \
+      ${TEST_UNRELIABLE} \
+      ${BACKENDS} \
+      --compare-previous
+
+   return $?
+}
+
+run_piglit_gl()
+{
+   UNRELIABLE="$1"
+
+   if [ $UNRELIABLE -eq 1 ]; then
+      TEST_UNRELIABLE="--unreliable"
+   fi
+
+   BACKENDS="--backend vtest-softpipe"
+   if [[ -z "$SOFTWARE_ONLY" ]]; then
+      BACKENDS="${BACKENDS} --backend vtest-gpu"
+   fi
 
-/virglrenderer/ci/run-deqp.sh --with-vtest $LIMIT_TESTSET
+   ./run_test_suite.sh --piglit --gles2 --gles3 \
-/virglrenderer/ci/run-deqp.sh --host-gl --with-vtest $LIMIT_TESTSET
+      --host-gl \
+      ${TEST_UNRELIABLE} \
+      ${BACKENDS} \
+      --compare-previous
 
-# Return test pass/fail
+   return $?
-if [ -f /virglrenderer/results/regressions_detected ]; then
+}
+
+run_piglit_gles()
+{
+   UNRELIABLE="$1"
+
+   if [ $UNRELIABLE -eq 1 ]; then
+      TEST_UNRELIABLE="--unreliable"
+   fi
+
+   BACKENDS="--backend vtest-softpipe"
+   if [[ -z "$SOFTWARE_ONLY" ]]; then
+      BACKENDS="${BACKENDS} --backend vtest-gpu"
+   fi
+
+   ./run_test_suite.sh --piglit --gles2 --gles3 \
+      --host-gles \
+      ${TEST_UNRELIABLE} \
+      ${BACKENDS} \
+      --compare-previous
+
+   return $?
+}
+
+parse_input()
+{
+   RET=0
+   while  [ -n "$1" ]; do
+      echo ""
+      UNRELIABLE=0
+
+      case $1 in
+         --make-check)
+         run_make_check
+         ;;
+
+         --deqp-gl-unreliable)
+         UNRELIABLE=1
+         ;& #Fallthrough
+
+         --deqp-gl)
+         run_deqp_gl $UNRELIABLE
+         ;;
+
+         --deqp-gles-unreliable)
+         UNRELIABLE=1
+         ;& #Fallthrough
+
+         --deqp-gles)
+         run_deqp_gles $UNRELIABLE
+         ;;
+
+         --piglit-gl-unreliable)
+         UNRELIABLE=1
+         ;& #Fallthrough
+
+         --piglit-gl)
+         run_piglit_gl $UNRELIABLE
+         ;;
+
+         --piglit-gles-unreliable)
+         UNRELIABLE=1
+         ;& #Fallthrough
+
+         --piglit-gles)
+         run_piglit_gles $UNRELIABLE
+         ;;
+
+         *)
+         echo "Unknown test option $1"
          exit 1
-fi
+      esac
+
+      if [ $? -ne 0 ]; then
+         RET=1
+      fi
+
+      shift
+   done
+
+   exit $RET
+}
+
+run_setup
+parse_input $@

ci/run_ci_locally.sh

@@ -41,4 +41,4 @@ time docker run \
      $LOCAL_MESA \
      $LOCAL_VIRGL \
      $DOCKER_IMAGE:latest \
-     /virglrenderer/ci/run-tests.sh 2>&1 | tee results/log.txt
+     bash -c "/virglrenderer/ci/run-tests.sh --make-check --deqp-gl --deqp-gles --piglit-gl --piglit-gles" 2>&1 | tee results/log.txt