virgl/gbm: improve gbm resource creation

This change makes a set of changes to increase the range of gbm-based
resources that can be allocated:
 - Add VIRGL_BIND_LINEAR to support linear gbm allocations.
 - Relax the bind flag argument check if VIRGL_BIND_SHARED or
   VIRGL_BIND_LINEAR is set.
 - For resources allocated from gbm, only try to create an image if one
   of the render target or sampler view bind flags is set.
 - Don't try to calculate the internal image format for external images.

This change also fixes a use-after-free that could occur if external
image creation failed.

Signed-off-by: David Stevens <stevensd@chromium.org>
Reviewed-by: Gurchetan Singh <gurchetansingh@chromium.org>

src/virgl_gbm.c

@@ -370,6 +370,9 @@ uint32_t virgl_gbm_convert_flags(uint32_t virgl_bind_flags)
       flags |= GBM_BO_USE_SCANOUT;
    if (virgl_bind_flags & VIRGL_BIND_CURSOR)
       flags |= GBM_BO_USE_CURSOR;
+   if (virgl_bind_flags & VIRGL_BIND_LINEAR)
+      flags |= GBM_BO_USE_LINEAR;
+
    return flags;
 }
 

src/virgl_hw.h

@@ -288,6 +288,8 @@ enum virgl_formats {
 
 #define VIRGL_BIND_PREFER_EMULATED_BGRA  (1 << 21)
 
+#define VIRGL_BIND_LINEAR (1 << 22)
+
 struct virgl_caps_bool_set1 {
         unsigned indep_blend_enable:1;
         unsigned indep_blend_func:1;

src/vrend_renderer.c

@@ -6099,8 +6099,11 @@ static int check_resource_valid(struct vrend_renderer_resource_create_args *args
       if (!((args->bind & VIRGL_BIND_SAMPLER_VIEW) ||
             (args->bind & VIRGL_BIND_DEPTH_STENCIL) ||
             (args->bind & VIRGL_BIND_RENDER_TARGET) ||
-            (args->bind & VIRGL_BIND_CURSOR)))
+            (args->bind & VIRGL_BIND_CURSOR) ||
+            (args->bind & VIRGL_BIND_SHARED) ||
+            (args->bind & VIRGL_BIND_LINEAR))) {
          return -1;
+      }
 
       if (args->target == PIPE_TEXTURE_2D ||
           args->target == PIPE_TEXTURE_RECT ||
@@ -6170,15 +6173,8 @@ static void *vrend_allocate_using_gbm(struct vrend_resource *gr)
    if (!bo)
       return NULL;
 
-   void *image = virgl_egl_image_from_dmabuf(egl, bo);
-   if (!image) {
-      gbm_bo_destroy(bo);
-      return NULL;
-   }
-
-   gr->egl_image = image;
    gr->gbm_bo = bo;
-   return image;
+   return bo;
 #else
    (void)gr;
    return NULL;
@@ -6197,8 +6193,19 @@ static int vrend_renderer_resource_allocate_texture(struct vrend_resource *gr,
    if (pr->width0 == 0)
       return EINVAL;
 
-   if (!image_oes)
+   if (!image_oes && vrend_allocate_using_gbm(gr)) {
-      image_oes = vrend_allocate_using_gbm(gr);
+      if ((gr->base.bind & (VIRGL_BIND_RENDER_TARGET | VIRGL_BIND_SAMPLER_VIEW)) == 0) {
+         gr->storage = VREND_RESOURCE_STORAGE_GBM_ONLY;
+         return 0;
+      }
+      image_oes = virgl_egl_image_from_dmabuf(egl, gr->gbm_bo);
+      if (!image_oes) {
+         gbm_bo_destroy(gr->gbm_bo);
+         gr->gbm_bo = NULL;
+      } else {
+         gr->egl_image = image_oes;
+      }
+   }
 
    bool format_can_texture_storage = has_feature(feat_texture_storage) &&
          (tex_conv_table[format].flags & VIRGL_TEXTURE_CAN_TEXTURE_STORAGE);
@@ -6242,16 +6249,6 @@ static int vrend_renderer_resource_allocate_texture(struct vrend_resource *gr,
 
    debug_texture(__func__, gr);
 
-   internalformat = tex_conv_table[format].internalformat;
-   glformat = tex_conv_table[format].glformat;
-   gltype = tex_conv_table[format].gltype;
-
-   if (internalformat == 0) {
-      vrend_printf("unknown format is %d\n", pr->format);
-      FREE(gt);
-      return EINVAL;
-   }
-
    glGenTextures(1, &gr->id);
    glBindTexture(gr->target, gr->id);
 
@@ -6260,11 +6257,23 @@ static int vrend_renderer_resource_allocate_texture(struct vrend_resource *gr,
          glEGLImageTargetTexture2DOES(gr->target, (GLeglImageOES) image_oes);
       } else {
          vrend_printf( "missing GL_OES_EGL_image_external extension\n");
+         glBindTexture(gr->target, 0);
          FREE(gr);
+         return EINVAL;
+      }
+   } else {
+      internalformat = tex_conv_table[format].internalformat;
+      glformat = tex_conv_table[format].glformat;
+      gltype = tex_conv_table[format].gltype;
+
+      if (internalformat == 0) {
+         vrend_printf("unknown format is %d\n", pr->format);
          glBindTexture(gr->target, 0);
+         FREE(gt);
          return EINVAL;
       }
-   } else if (pr->nr_samples > 0) {
+
+      if (pr->nr_samples > 0) {
          if (format_can_texture_storage) {
             if (gr->target == GL_TEXTURE_2D_MULTISAMPLE) {
                glTexStorage2DMultisample(gr->target, pr->nr_samples,
@@ -6345,6 +6354,7 @@ static int vrend_renderer_resource_allocate_texture(struct vrend_resource *gr,
             }
          }
       }
+   }
 
    if (!format_can_texture_storage) {
       glTexParameteri(gr->target, GL_TEXTURE_BASE_LEVEL, 0);

src/vrend_renderer.h

@@ -58,6 +58,7 @@ enum vrend_resource_storage_type {
    /* The resource contents are stored in shared guest memory if it's
     * attached, otherwise in host system memory. */
    VREND_RESOURCE_STORAGE_GUEST_ELSE_SYSTEM,
+   VREND_RESOURCE_STORAGE_GBM_ONLY,
 };
 
 struct vrend_resource {