mesa: restrict use of GL_ABGR_EXT format to allowed data types

GL_UNSIGNED_SHORT_5_5_5_1, GL_UNSIGNED_SHORT_1_5_5_5_REV,
GL_UNSIGNED_INT_10_10_10_2, GL_UNSIGNED_INT_2_10_10_10_REV data types
are not explicitly allowed to work with GL_ABGR_EXT format neither
in GL nor GL_EXT_abgr specs.

Removed the corresponding mesa formats as there are no other functions
using them inside Mesa anymore.

Signed-off-by: Samuel Iglesias Gonsalvez <[email protected]>
Reviewed-by: Jason Ekstrand <[email protected]>

4 files changed, 46 insertions, 52 deletions

diff --git a/src/mesa/main/formats.c b/src/mesa/main/formats.c
index a2df2859d91..8ac0583c027 100644
--- a/src/mesa/main/formats.c
+++ b/src/mesa/main/formats.c
@@ -1023,18 +1023,10 @@ _mesa_format_to_type_and_comps(mesa_format format,
       *datatype = GL_UNSIGNED_SHORT_4_4_4_4;
       *comps = 4;
       return;
-   case MESA_FORMAT_R1G5B5A5_UNORM:
-      *datatype = GL_UNSIGNED_SHORT_5_5_5_1;
-      *comps = 4;
-      return;
    case MESA_FORMAT_R5G5B5A1_UNORM:
       *datatype = GL_UNSIGNED_SHORT_1_5_5_5_REV;
       *comps = 4;
       return;
-   case MESA_FORMAT_A5B5G5R1_UNORM:
-      *datatype = GL_UNSIGNED_SHORT_1_5_5_5_REV;
-      *comps = 4;
-      return;
    case MESA_FORMAT_A2B10G10R10_UNORM:
    case MESA_FORMAT_A2B10G10R10_UINT:
       *datatype = GL_UNSIGNED_INT_10_10_10_2;
@@ -1045,14 +1037,6 @@ _mesa_format_to_type_and_comps(mesa_format format,
       *datatype = GL_UNSIGNED_INT_10_10_10_2;
       *comps = 4;
       return;
-   case MESA_FORMAT_R2G10B10A10_UNORM:
-      *datatype = GL_UNSIGNED_INT_10_10_10_2;
-      *comps = 4;
-      return;
-   case MESA_FORMAT_A10B10G10R2_UNORM:
-      *datatype = GL_UNSIGNED_INT_2_10_10_10_REV;
-      *comps = 4;
-      return;
 
    case MESA_FORMAT_B2G3R3_UNORM:
       *datatype = GL_UNSIGNED_BYTE_3_3_2;
@@ -1730,15 +1714,9 @@ _mesa_format_matches_format_and_type(mesa_format mesa_format,
 
       return GL_FALSE;
 
-   case MESA_FORMAT_R1G5B5A5_UNORM:
-      return format == GL_ABGR_EXT && type == GL_UNSIGNED_SHORT_5_5_5_1;
-
    case MESA_FORMAT_R5G5B5A1_UNORM:
       return format == GL_RGBA && type == GL_UNSIGNED_SHORT_1_5_5_5_REV;
 
-   case MESA_FORMAT_A5B5G5R1_UNORM:
-      return format == GL_ABGR_EXT && type == GL_UNSIGNED_SHORT_1_5_5_5_REV;
-
    case MESA_FORMAT_A2B10G10R10_UNORM:
       return format == GL_RGBA && type == GL_UNSIGNED_INT_10_10_10_2;
 
@@ -1751,12 +1729,6 @@ _mesa_format_matches_format_and_type(mesa_format mesa_format,
    case MESA_FORMAT_A2R10G10B10_UINT:
       return format == GL_BGRA_INTEGER_EXT && type == GL_UNSIGNED_INT_10_10_10_2;
 
-   case MESA_FORMAT_R2G10B10A10_UNORM:
-      return format == GL_ABGR_EXT && type == GL_UNSIGNED_INT_10_10_10_2;
-
-   case MESA_FORMAT_A10B10G10R2_UNORM:
-      return format == GL_ABGR_EXT && type == GL_UNSIGNED_INT_2_10_10_10_REV;
-
    case MESA_FORMAT_A_UNORM8:
       return format == GL_ALPHA && type == GL_UNSIGNED_BYTE;
    case MESA_FORMAT_A_UNORM16:
diff --git a/src/mesa/main/formats.csv b/src/mesa/main/formats.csv
index 09fc9b34776..15d82aa5911 100644
--- a/src/mesa/main/formats.csv
+++ b/src/mesa/main/formats.csv
@@ -91,13 +91,9 @@ MESA_FORMAT_Z24_UNORM_X8_UINT             , packed, 1, 1, x8  , un24,     ,
 MESA_FORMAT_R3G3B2_UNORM                  , packed, 1, 1, un3 , un3 , un2 ,     , xyz1, rgb
 MESA_FORMAT_A4B4G4R4_UNORM                , packed, 1, 1, un4 , un4 , un4 , un4 , wzyx, rgb
 MESA_FORMAT_R4G4B4A4_UNORM                , packed, 1, 1, un4 , un4 , un4 , un4 , xyzw, rgb
-MESA_FORMAT_R1G5B5A5_UNORM                , packed, 1, 1, un1 , un5 , un5 , un5 , xyzw, rgb
 MESA_FORMAT_R5G5B5A1_UNORM                , packed, 1, 1, un5 , un5 , un5 , un1 , xyzw, rgb
-MESA_FORMAT_A5B5G5R1_UNORM                , packed, 1, 1, un5 , un5 , un5 , un1 , wzyx, rgb
 MESA_FORMAT_A2B10G10R10_UNORM             , packed, 1, 1, un2 , un10, un10, un10, wzyx, rgb
 MESA_FORMAT_A2R10G10B10_UNORM             , packed, 1, 1, un2 , un10, un10, un10, yzwx, rgb
-MESA_FORMAT_R2G10B10A10_UNORM             , packed, 1, 1, un2 , un10, un10, un10, xyzw, rgb
-MESA_FORMAT_A10B10G10R2_UNORM             , packed, 1, 1, un10, un10, un10, un2 , wzyx, rgb
 
 MESA_FORMAT_YCBCR                         , other , 1, 1, x16 ,     ,     ,     , xyzw, yuv
 MESA_FORMAT_YCBCR_REV                     , other , 1, 1, x16 ,     ,     ,     , xyzw, yuv
diff --git a/src/mesa/main/formats.h b/src/mesa/main/formats.h
index 02afbce7c4f..09890ecc8b2 100644
--- a/src/mesa/main/formats.h
+++ b/src/mesa/main/formats.h
@@ -362,14 +362,9 @@ typedef enum
    MESA_FORMAT_R3G3B2_UNORM,                                   /* BBGG GRRR */
    MESA_FORMAT_A4B4G4R4_UNORM,                       /* RRRR GGGG BBBB AAAA */
    MESA_FORMAT_R4G4B4A4_UNORM,                       /* AAAA BBBB GGGG RRRR */
-   MESA_FORMAT_R1G5B5A5_UNORM,                       /* AAAA ABBB BBGG GGGR */
    MESA_FORMAT_R5G5B5A1_UNORM,                       /* ABBB BBGG GGGR RRRR */
-   MESA_FORMAT_A5B5G5R1_UNORM,                       /* RGGG GGBB BBBA AAAA */
    MESA_FORMAT_A2B10G10R10_UNORM,/* RRRR RRRR RRGG GGGG GGGG BBBB BBBB BBAA */
    MESA_FORMAT_A2R10G10B10_UNORM,/* BBBB BBBB BBGG GGGG GGGG RRRR RRRR RRAA */
-   MESA_FORMAT_R2G10B10A10_UNORM,/* AAAA AAAA AABB BBBB BBBB GGGG GGGG GGRR */
-   MESA_FORMAT_A10B10G10R2_UNORM,/* RRGG GGGG GGGG BBBB BBBB BBAA AAAA AAAA */
-
 
    MESA_FORMAT_YCBCR,            /*                     YYYY YYYY UorV UorV */
    MESA_FORMAT_YCBCR_REV,        /*                     UorV UorV YYYY YYYY */
diff --git a/src/mesa/main/glformats.c b/src/mesa/main/glformats.c
index ca0b62d4b3c..3e913ed988c 100644
--- a/src/mesa/main/glformats.c
+++ b/src/mesa/main/glformats.c
@@ -456,18 +456,29 @@ _mesa_bytes_per_pixel(GLenum format, GLenum type)
          return -1;  /* error */
    case GL_UNSIGNED_SHORT_4_4_4_4:
    case GL_UNSIGNED_SHORT_4_4_4_4_REV:
+      if (format == GL_RGBA || format == GL_BGRA || format == GL_ABGR_EXT ||
+          format == GL_RGBA_INTEGER_EXT || format == GL_BGRA_INTEGER_EXT)
+         return sizeof(GLushort);
+      else
+         return -1;
    case GL_UNSIGNED_SHORT_5_5_5_1:
    case GL_UNSIGNED_SHORT_1_5_5_5_REV:
-      if (format == GL_RGBA || format == GL_BGRA || format == GL_ABGR_EXT ||
+      if (format == GL_RGBA || format == GL_BGRA ||
           format == GL_RGBA_INTEGER_EXT || format == GL_BGRA_INTEGER_EXT)
          return sizeof(GLushort);
       else
          return -1;
    case GL_UNSIGNED_INT_8_8_8_8:
    case GL_UNSIGNED_INT_8_8_8_8_REV:
+      if (format == GL_RGBA || format == GL_BGRA || format == GL_ABGR_EXT ||
+          format == GL_RGBA_INTEGER_EXT || format == GL_BGRA_INTEGER_EXT ||
+          format == GL_RGB)
+         return sizeof(GLuint);
+      else
+         return -1;
    case GL_UNSIGNED_INT_10_10_10_2:
    case GL_UNSIGNED_INT_2_10_10_10_REV:
-      if (format == GL_RGBA || format == GL_BGRA || format == GL_ABGR_EXT ||
+      if (format == GL_RGBA || format == GL_BGRA ||
           format == GL_RGBA_INTEGER_EXT || format == GL_BGRA_INTEGER_EXT ||
           format == GL_RGB)
          return sizeof(GLuint);
@@ -1601,12 +1612,8 @@ _mesa_error_check_format_and_type(const struct gl_context *ctx,
 
    case GL_UNSIGNED_SHORT_4_4_4_4:
    case GL_UNSIGNED_SHORT_4_4_4_4_REV:
-   case GL_UNSIGNED_SHORT_5_5_5_1:
-   case GL_UNSIGNED_SHORT_1_5_5_5_REV:
    case GL_UNSIGNED_INT_8_8_8_8:
    case GL_UNSIGNED_INT_8_8_8_8_REV:
-   case GL_UNSIGNED_INT_10_10_10_2:
-   case GL_UNSIGNED_INT_2_10_10_10_REV:
       if (format == GL_RGBA ||
           format == GL_BGRA ||
           format == GL_ABGR_EXT) {
@@ -1616,6 +1623,20 @@ _mesa_error_check_format_and_type(const struct gl_context *ctx,
           ctx->Extensions.ARB_texture_rgb10_a2ui) {
          break; /* OK */
       }
+      return GL_INVALID_OPERATION;
+
+   case GL_UNSIGNED_SHORT_5_5_5_1:
+   case GL_UNSIGNED_SHORT_1_5_5_5_REV:
+   case GL_UNSIGNED_INT_10_10_10_2:
+   case GL_UNSIGNED_INT_2_10_10_10_REV:
+      if (format == GL_RGBA ||
+          format == GL_BGRA) {
+         break; /* OK */
+      }
+      if ((format == GL_RGBA_INTEGER_EXT || format == GL_BGRA_INTEGER_EXT) &&
+          ctx->Extensions.ARB_texture_rgb10_a2ui) {
+         break; /* OK */
+      }
       if (type == GL_UNSIGNED_INT_2_10_10_10_REV && format == GL_RGB &&
           ctx->API == API_OPENGLES2) {
          break; /* OK by GL_EXT_texture_type_2_10_10_10_REV */
@@ -1759,7 +1780,6 @@ _mesa_error_check_format_and_type(const struct gl_context *ctx,
 
       case GL_RGBA:
       case GL_BGRA:
-      case GL_ABGR_EXT:
          switch (type) {
             case GL_BYTE:
             case GL_UNSIGNED_BYTE:
@@ -1782,6 +1802,25 @@ _mesa_error_check_format_and_type(const struct gl_context *ctx,
                return GL_INVALID_ENUM;
          }
 
+      case GL_ABGR_EXT:
+         switch (type) {
+            case GL_BYTE:
+            case GL_UNSIGNED_BYTE:
+            case GL_SHORT:
+            case GL_UNSIGNED_SHORT:
+            case GL_INT:
+            case GL_UNSIGNED_INT:
+            case GL_FLOAT:
+            case GL_UNSIGNED_SHORT_4_4_4_4:
+            case GL_UNSIGNED_SHORT_4_4_4_4_REV:
+            case GL_UNSIGNED_INT_8_8_8_8:
+            case GL_UNSIGNED_INT_8_8_8_8_REV:
+            case GL_HALF_FLOAT:
+               return GL_NO_ERROR;
+            default:
+               return GL_INVALID_ENUM;
+         }
+
       case GL_YCBCR_MESA:
          if (!ctx->Extensions.MESA_ycbcr_texture)
             return GL_INVALID_ENUM;
@@ -2597,16 +2636,12 @@ _mesa_format_from_format_and_type(GLenum format, GLenum type)
          return MESA_FORMAT_A1B5G5R5_UNORM;
       else if (format == GL_BGRA)
          return MESA_FORMAT_A1R5G5B5_UNORM;
-      else if (format == GL_ABGR_EXT)
-         return MESA_FORMAT_R1G5B5A5_UNORM;
       break;
    case GL_UNSIGNED_SHORT_1_5_5_5_REV:
       if (format == GL_RGBA)
          return MESA_FORMAT_R5G5B5A1_UNORM;
       else if (format == GL_BGRA)
          return MESA_FORMAT_B5G5R5A1_UNORM;
-      else if (format == GL_ABGR_EXT)
-         return MESA_FORMAT_A5B5G5R1_UNORM;
       break;
    case GL_UNSIGNED_BYTE_3_3_2:
       if (format == GL_RGB)
@@ -2629,8 +2664,6 @@ _mesa_format_from_format_and_type(GLenum format, GLenum type)
          return MESA_FORMAT_A2R10G10B10_UNORM;
       else if (format == GL_BGRA_INTEGER)
          return MESA_FORMAT_A2R10G10B10_UINT;
-      else if (format == GL_ABGR_EXT)
-         return MESA_FORMAT_R2G10B10A10_UNORM;
       break;
    case GL_UNSIGNED_INT_2_10_10_10_REV:
       if (format == GL_RGBA)
@@ -2641,8 +2674,6 @@ _mesa_format_from_format_and_type(GLenum format, GLenum type)
          return MESA_FORMAT_B10G10R10A2_UNORM;
       else if (format == GL_BGRA_INTEGER)
          return MESA_FORMAT_B10G10R10A2_UINT;
-      else if (format == GL_ABGR_EXT)
-         return MESA_FORMAT_A10B10G10R2_UNORM;
       break;
    case GL_UNSIGNED_INT_8_8_8_8:
       if (format == GL_RGBA)