virglrenderer/ci/run_test_suite.sh

#!/bin/bash

# Setup paths and import util functions
. $(dirname $(readlink -f "$0"))/util.sh

TESTS=""
BACKENDS=""
NUM_THREADS=${NUM_THREADS:-$(expr $(expr $(nproc) / 8) + 1)}

COMPARE_BACKENDS=0
COMPARE_PREVIOUS=0
USE_HOST_GLES=0
TEST_APP="deqp"
VERIFY_UNRELIABLE_TESTS=0

parse_input()
{
   while  [ -n "$1" ]; do
      case $1 in

      -u|--unreliable)
         VERIFY_UNRELIABLE_TESTS=1
         ;;

      -a|--all-backends)
         BACKENDS=""
         BACKENDS="$BACKENDS vtest-softpipe"
         BACKENDS="$BACKENDS vtest-llvmpipe"
         BACKENDS="$BACKENDS vtest-gpu"
         BACKENDS="$BACKENDS softpipe"
         BACKENDS="$BACKENDS llvmpipe"
         BACKENDS="$BACKENDS gpu"
         ;;

   	-v|--vtest)
         BACKENDS=""
         BACKENDS="$BACKENDS vtest-softpipe"
         BACKENDS="$BACKENDS vtest-llvmpipe"
         BACKENDS="$BACKENDS vtest-gpu"
         ;;

      --host-gles)
         USE_HOST_GLES=1
         ;;

      --host-gl)
         USE_HOST_GLES=0
         ;;

      -b|--backend)
         NEW_BACKEND="$2"
         shift
         BACKENDS="$BACKENDS $NEW_BACKEND"
         ;;

      -j|--threads)
         NUM_THREADS=$2
         shift
         ;;

   	--gles2)
         TESTS="$TESTS gles2"
   	   ;;

   	--gles3)
         TESTS="$TESTS gles3"
   	   ;;

   	--gles31)
         TESTS="$TESTS gles31"
   	   ;;

      --gl30)
         TESTS="$TESTS gl30"
         ;;

      --gl31)
         TESTS="$TESTS gl31"
         ;;

      --gl32)
         TESTS="$TESTS gl32"
         ;;

      -d|--deqp)
         TEST_APP="deqp"
         ;;

      -p|--piglit)
         TEST_APP="piglit"
         ;;

      -cp|--compare-previous)
         # Compare results against previous runs
         COMPARE_PREVIOUS=1
         ;;

   	*)
   	   echo "Unknown flag $1"
   	   exit 1
      esac
      shift
   done

   if [[ -z $BACKENDS ]]; then
      BACKENDS="gpu"
   fi

   # These two options are incompatible, and one has to be disabled
   if [ $VERIFY_UNRELIABLE_TESTS -eq 1 ]; then
      COMPARE_PREVIOUS=0
   fi
}

compare_previous()
{
   if [ ! -f $PREVIOUS_RESULTS_DIR/results.txt ]; then
      return 2
   fi

   # Piglit tests use @ as separator for path/to/test
   IGNORE_TESTS=$(sed "s/\@/\//g" $IGNORE_TESTS_FILE 2>/dev/null)

   # Avoid introducing changes while doing this comparison
   TMP_RESULTS=$(mktemp /tmp/virgl_ci_results.XXXXXX)
   TMP_PREV_RESULTS=$(mktemp /tmp/virgl_ci_previous_results.XXXXXX)
   cp $RESULTS_DIR/results.txt $TMP_RESULTS
   cp $PREVIOUS_RESULTS_DIR/results.txt $TMP_PREV_RESULTS

   for TEST in $IGNORE_TESTS; do
      sed -i "\:$TEST:d" $TMP_RESULTS $TMP_PREV_RESULTS
   done

   if [ "$TEST_APP" = "piglit" ]; then
      # This distinction adds too much variability
      sed -i "s/crash/fail/g" $TMP_RESULTS $TMP_PREV_RESULTS
   elif [ "$TEST_APP" = "deqp" ]; then
      # This distinction adds too much variability
      sed -i "s/QualityWarning/Pass/g" $TMP_RESULTS $TMP_PREV_RESULTS
      sed -i "s/CompatibilityWarning/Pass/g" $TMP_RESULTS $TMP_PREV_RESULTS
   fi

   # Sort results files
   sort -V $TMP_RESULTS -o $TMP_RESULTS
   sort -V $TMP_PREV_RESULTS -o $TMP_PREV_RESULTS

   diff -u $TMP_PREV_RESULTS $TMP_RESULTS 2>&1 > $RESULTS_DIR/regression_diff.txt
   if [ $? -ne 0 ]; then
      touch $VIRGL_PATH/results/regressions_detected
      return 1
   else
      rm -rf $RESULTS_DIR/regression_diff.txt
      return 0
   fi
}

interpret_results()
{
   PASSED_TESTS="$1"
   TOTAL_TESTS="$2"

   if [ $COMPARE_PREVIOUS -eq 1 ]; then
      compare_previous
      case $? in
         0)
         echo "Pass - matches previous results"
         return 0
         ;;

         1)
         echo "Fail - diff against previous results: $RESULTS_DIR/regression_diff.txt"
         return 1
         ;;

         2)
         echo "Pass - no previous results, but passed $PASSED_TESTS/$TOTAL_TESTS tests"
         return 0
         ;;

         *)
         echo "BUG!"
         return 1
         ;;
      esac
   else
      if [ $PASSED_TESTS -eq $TOTAL_TESTS ] && [ $TOTAL_TESTS -ne 0 ]; then
         echo "Pass - passed $PASSED_TESTS/$TOTAL_TESTS tests"
         return 0
      else
         echo "Fail - passed $PASSED_TESTS/$TOTAL_TESTS tests: $RESULTS_DIR/results.txt"
         return 1
      fi
   fi
}

run_vtest_server()
{
   (
   if [ $USE_HOST_GLES -eq 1 ]; then
      VTEST_USE_EGL_SURFACELESS=1 \
      VTEST_USE_GLES=1 \
      virgl_test_server &>$VTEST_LOG_FILE &
   else
      VTEST_USE_EGL_SURFACELESS=1 \
      virgl_test_server &>$VTEST_LOG_FILE &
   fi
   )
}

run_test_suite()
{
   local BACKEND="$1"
   local TEST_NAME="$2"

#   echo "run_test_suite() OUTPUT_PATH: $OUTPUT_PATH"
#   echo "run_test_suite() LOG_FILE: $LOG_FILE"
#   echo "run_test_suite() RESULTS_FILE: $RESULTS_FILE"

   if [ $VERIFY_UNRELIABLE_TESTS -eq 1 ]; then
      UNRELIABLE_STRING="unreliable "
   fi

   if [[ $BACKEND == *"vtest"* ]]; then
      printf "Running ${UNRELIABLE_STRING}$TEST_APP-$TEST_NAME on vtest-$DRIVER_NAME: "
   else
      printf "Running ${UNRELIABLE_STRING}$TEST_APP-$TEST_NAME on $DRIVER_NAME: "
   fi

   if test $VERIFY_UNRELIABLE_TESTS -eq 1; then
      TEST_FILE="$IGNORE_TESTS_FILE"
      if test ! -f $TEST_FILE -o $(wc -l $TEST_FILE | cut -f1 -d' ') -eq 0; then
         echo "Unreliable: no ignore tests."
         return 0
      fi
   fi

   case $TEST_APP in
   piglit)
      # Don't run GLX tests
      PIGLIT_TESTS=" -x glx"

      if [ $VERIFY_UNRELIABLE_TESTS -eq 1 ]; then
         # XXX: Fold the glx exception?
         PIGLIT_TESTS_CMD="--test-list $TEST_FILE"
      else
         # TODO: create test_file for normal runs
         PIGLIT_TESTS_CMD="$PIGLIT_TESTS -t $TEST_NAME"

      fi

      piglit run --platform x11_egl \
         -l verbose \
         --jobs $NUM_THREADS \
         $PIGLIT_TESTS_CMD \
         gpu \
         /tmp/  &> $LOG_FILE

      piglit summary console /tmp/ | grep -B 999999 "summary:" | grep -v "summary:" > $RESULTS_FILE

      TOTAL_TESTS=$(cat $RESULTS_FILE | wc -l)
      PASSED_TESTS=$(grep " pass" $RESULTS_FILE | wc -l)
      ;;

   deqp)
      deqp  \
         --cts-build-dir $CTS_PATH/build \
         --test-names-file "$TEST_FILE" \
         --results-file "$RESULTS_FILE" \
         --threads $NUM_THREADS &> $LOG_FILE

#      echo "$(which dEQP): Returned $?"
#      echo "ls -la CTS_PATH/build: $CTS_PATH/build - $(ls -la $CTS_PATH/build)"
#      echo "ls -la TEST_FILE: $TEST_FILE - $(ls -la $TEST_FILE)"
#      echo "ls -la RESULTS_FILE: $RESULTS_FILE - $(ls -la $RESULTS_FILE)"
#      echo "LOG_FILE: $LOG_FILE - $(ls -la $LOG_FILE)"

      # Remove header
      sed -i "/#/d" $RESULTS_FILE

      # Sort results file to make diffs easier to read
      sort -V $RESULTS_FILE -o $RESULTS_FILE

      TOTAL_TESTS=$(cat $RESULTS_FILE | wc -l)
      PASSED_TESTS=$(grep " Pass" $RESULTS_FILE | wc -l)
      ;;
   esac

   interpret_results $PASSED_TESTS $TOTAL_TESTS $UNRELIABLE
   return $?
}

create_result_dir()
{
   if [[ -n $GALLIUM_DRIVER ]]; then
      HOST_DRIVER="_${GALLIUM_DRIVER}"
   elif [[ -n $DRIVER_NAME ]]; then
      HOST_DRIVER="_${DRIVER_NAME}"
   fi

   if [ $USE_HOST_GLES -eq 0 ]; then
      HOST_GL="gl"
   else
      HOST_GL="es"
   fi

   TEST_PATH=${HOST_GL}_host${HOST_DRIVER}/${TEST_APP}_${TEST_NAME}
   RESULTS_DIR=$VIRGL_PATH/results/${TEST_PATH}
   PREVIOUS_RESULTS_DIR=$VIRGL_PATH/ci/previous_results/${TEST_PATH}
   IGNORE_TESTS_FILE=$PREVIOUS_RESULTS_DIR/ignore_tests.txt

   # Remove comments from test-list
   FILTERED_TEST_FILE=$(mktemp /tmp/virgl-ci.XXXXX)
   sed '/^#/d;/^$/d' $IGNORE_TESTS_FILE 2>/dev/null > $FILTERED_TEST_FILE
   IGNORE_TESTS_FILE=$FILTERED_TEST_FILE

   mkdir -p "$RESULTS_DIR"

   export OUTPUT_PATH="${RESULTS_DIR}"
   export RESULTS_FILE="${OUTPUT_PATH}/results.txt"
   export LOG_FILE="${OUTPUT_PATH}/log.txt"
   export VTEST_LOG_FILE="${OUTPUT_PATH}/vtest_log.txt"
}

run_test_on_backends()
{
   local BACKENDS="$1"
   local TEST_NAME="$2"
   local TEST_FILE="$3"
   local RET=0

#   echo "run_test_on_backends() BACKENDS: $BACKENDS"
#   echo "run_test_on_backends() TEST_NAME: $TEST_NAME"
#   echo "run_test_on_backends() TEST_FILE: $TEST_FILE"

   for BACKEND in $BACKENDS; do
         unset DRIVER_NAME
         unset GALLIUM_DRIVER
         unset GALLIVM_PERF
         unset LIBGL_ALWAYS_SOFTWARE
         unset VTEST_USE_EGL_SURFACELESS

         case $BACKEND in
            vtest-softpipe|softpipe)
               export LIBGL_ALWAYS_SOFTWARE=1
               export GALLIUM_DRIVER=softpipe
               export DRIVER_NAME=$GALLIUM_DRIVER
               ;;

            vtest-llvmpipe|llvmpipe)
               export GALLIVM_PERF=nopt,no_filter_hacks
               export LIBGL_ALWAYS_SOFTWARE=1
               export GALLIUM_DRIVER=llvmpipe
               export DRIVER_NAME=$GALLIUM_DRIVER
               ;;

            vtest-gpu|gpu)
               DEVICE_NAME=$(basename /dev/dri/renderD128)
               export DRIVER_NAME="$(basename `readlink /sys/class/drm/${DEVICE_NAME}/device/driver`)"
               ;;
         esac

         # This case statement is broken into two parts
         # because for the second part the LOG_FILE has
         # declared, which is needed to redirect FDs
         create_result_dir

         case $BACKEND in
            vtest-*)
               run_vtest_server
               export GALLIUM_DRIVER=virpipe
               ;;

            *)
               ;;
         esac

         run_test_suite "$BACKEND" "$TEST_NAME"

         if [ $? -ne 0 ]; then
            RET=1
         fi

         killall -q virgl_test_server
   done

   return $RET
}

run_all_tests()
{
   local BACKENDS=$1
   local TESTS=$2
   local RET=0

   if [ $USE_HOST_GLES -eq 0 ]; then
      echo "Running test(s) on GL Host"
      echo "--------------------------"
   else
      echo "Running test(s) on GLES Host"
      echo "----------------------------"
   fi
#   echo "run_all_tests() BACKENDS: $BACKENDS"
#   echo "run_all_tests() TESTS: $TESTS"


   # TODO: add similar must pass lists for piglit
   for TEST in $TESTS; do
      case $TEST in
      gles2|gles3|gles31)
         TEST_FILE="$CTS_PATH/android/cts/master/$TEST-master.txt"
         ;;
      gl30|gl31|gl32)
         TEST_FILE="$CTS_PATH/external/openglcts/data/mustpass/gl/khronos_mustpass/4.6.1.x/$TEST-master.txt"
         ;;
      esac

      run_test_on_backends "$BACKENDS" "$TEST" "$TEST_FILE"

      if [ $? -ne 0 ]; then
         RET=1
      fi
   done

   exit $RET
}

setup()
{
   Xvfb :0 -screen 0 1024x768x24 &>/dev/null &
   export DISPLAY=:0
   sleep 2
}

setup
parse_input $@
run_all_tests "$BACKENDS" "$TESTS"