vl/csc: simplify matrix handling

A csc matrix is only 4x3 not 4x4, also define a VDPAU compatible type for it.

Signed-off-by: Christian König <[email protected]>

8 files changed, 82 insertions, 85 deletions

diff --git a/src/gallium/auxiliary/vl/vl_compositor.c b/src/gallium/auxiliary/vl/vl_compositor.c
index 94d937714ba..35958360e41 100644
--- a/src/gallium/auxiliary/vl/vl_compositor.c
+++ b/src/gallium/auxiliary/vl/vl_compositor.c
@@ -52,8 +52,6 @@ enum VS_OUTPUT
    VS_O_VBOTTOM,
 };
 
-typedef float csc_matrix[16];
-
 static void *
 create_vert_shader(struct vl_compositor *c)
 {
@@ -804,7 +802,7 @@ vl_compositor_cleanup(struct vl_compositor *c)
 }
 
 void
-vl_compositor_set_csc_matrix(struct vl_compositor_state *s, const float matrix[16])
+vl_compositor_set_csc_matrix(struct vl_compositor_state *s, vl_csc_matrix const *matrix)
 {
    struct pipe_transfer *buf_transfer;
 
@@ -816,7 +814,7 @@ vl_compositor_set_csc_matrix(struct vl_compositor_state *s, const float matrix[1
                       PIPE_TRANSFER_WRITE | PIPE_TRANSFER_DISCARD_RANGE,
                       &buf_transfer),
       matrix,
-      sizeof(csc_matrix)
+      sizeof(vl_csc_matrix)
    );
 
    pipe_buffer_unmap(s->pipe, buf_transfer);
@@ -1052,7 +1050,7 @@ vl_compositor_init(struct vl_compositor *c, struct pipe_context *pipe)
 bool
 vl_compositor_init_state(struct vl_compositor_state *s, struct pipe_context *pipe)
 {
-   csc_matrix csc_matrix;
+   vl_csc_matrix csc_matrix;
 
    assert(s);
 
@@ -1078,8 +1076,8 @@ vl_compositor_init_state(struct vl_compositor_state *s, struct pipe_context *pip
 
    vl_compositor_clear_layers(s);
 
-   vl_csc_get_matrix(VL_CSC_COLOR_STANDARD_IDENTITY, NULL, true, csc_matrix);
-   vl_compositor_set_csc_matrix(s, csc_matrix);
+   vl_csc_get_matrix(VL_CSC_COLOR_STANDARD_IDENTITY, NULL, true, &csc_matrix);
+   vl_compositor_set_csc_matrix(s, (const vl_csc_matrix *)&csc_matrix);
 
    return true;
 }
diff --git a/src/gallium/auxiliary/vl/vl_compositor.h b/src/gallium/auxiliary/vl/vl_compositor.h
index f8f3dc0de98..6de6ca06521 100644
--- a/src/gallium/auxiliary/vl/vl_compositor.h
+++ b/src/gallium/auxiliary/vl/vl_compositor.h
@@ -35,6 +35,7 @@
 #include "util/u_rect.h"
 
 #include "vl_types.h"
+#include "vl_csc.h"
 
 struct pipe_context;
 
@@ -126,7 +127,7 @@ vl_compositor_init_state(struct vl_compositor_state *state, struct pipe_context
  * set yuv -> rgba conversion matrix
  */
 void
-vl_compositor_set_csc_matrix(struct vl_compositor_state *settings, const float mat[16]);
+vl_compositor_set_csc_matrix(struct vl_compositor_state *settings, const vl_csc_matrix *matrix);
 
 /**
  * reset dirty area, so it's cleared with the clear colour
diff --git a/src/gallium/auxiliary/vl/vl_csc.c b/src/gallium/auxiliary/vl/vl_csc.c
index 56e662157ef..2ce690755f7 100644
--- a/src/gallium/auxiliary/vl/vl_csc.c
+++ b/src/gallium/auxiliary/vl/vl_csc.c
@@ -100,12 +100,11 @@
  * Y is in [16,235], Cb and Cr are in [16,240]
  * R, G, and B are in [16,235]
  */
-static const float bt_601[16] =
+static const vl_csc_matrix bt_601 =
 {
-   1.0f,  0.0f,    1.371f, 0.0f,
-   1.0f, -0.336f, -0.698f, 0.0f,
-   1.0f,  1.732f,  0.0f,   0.0f,
-   0.0f,  0.0f,    0.0f,   1.0f
+   { 1.0f,  0.0f,    1.371f, 0.0f, },
+   { 1.0f, -0.336f, -0.698f, 0.0f, },
+   { 1.0f,  1.732f,  0.0f,   0.0f, }
 };
 
 /*
@@ -113,12 +112,11 @@ static const float bt_601[16] =
  * Y is in [16,235], Cb and Cr are in [16,240]
  * R, G, and B are in [0,255]
  */
-static const float bt_601_full[16] =
+static const vl_csc_matrix bt_601_full =
 {
-   1.164f,  0.0f,    1.596f, 0.0f,
-   1.164f, -0.391f, -0.813f, 0.0f,
-   1.164f,  2.018f,  0.0f,   0.0f,
-   0.0f,    0.0f,    0.0f,   1.0f
+   { 1.164f,  0.0f,    1.596f, 0.0f, },
+   { 1.164f, -0.391f, -0.813f, 0.0f, },
+   { 1.164f,  2.018f,  0.0f,   0.0f, }
 };
 
 /*
@@ -126,12 +124,11 @@ static const float bt_601_full[16] =
  * Y is in [16,235], Cb and Cr are in [16,240]
  * R, G, and B are in [16,235]
  */
-static const float bt_709[16] =
+static const vl_csc_matrix bt_709 =
 {
-   1.0f,  0.0f,    1.540f, 0.0f,
-   1.0f, -0.183f, -0.459f, 0.0f,
-   1.0f,  1.816f,  0.0f,   0.0f,
-   0.0f,  0.0f,    0.0f,   1.0f
+   { 1.0f,  0.0f,    1.540f, 0.0f, },
+   { 1.0f, -0.183f, -0.459f, 0.0f, },
+   { 1.0f,  1.816f,  0.0f,   0.0f, }
 };
 
 /*
@@ -139,36 +136,32 @@ static const float bt_709[16] =
  * Y is in [16,235], Cb and Cr are in [16,240]
  * R, G, and B are in [0,255]
  */
-static const float bt_709_full[16] =
+static const vl_csc_matrix bt_709_full =
 {
-   1.164f,  0.0f,    1.793f, 0.0f,
-   1.164f, -0.213f, -0.534f, 0.0f,
-   1.164f,  2.115f,  0.0f,   0.0f,
-   0.0f,    0.0f,    0.0f,   1.0f
+   { 1.164f,  0.0f,    1.793f, 0.0f, },
+   { 1.164f, -0.213f, -0.534f, 0.0f, },
+   { 1.164f,  2.115f,  0.0f,   0.0f, }
 };
 
-static const float smpte240m[16] =
+static const vl_csc_matrix smpte240m =
 {
-   1.0f,  0.0f,    1.582f, 0.0f,
-   1.0f, -0.228f, -0.478f, 0.0f,
-   1.0f,  1.833f,  0.0f,   0.0f,
-   0.0f,  0.0f,    0.0f,   1.0f
+   { 1.0f,  0.0f,    1.582f, 0.0f, },
+   { 1.0f, -0.228f, -0.478f, 0.0f, },
+   { 1.0f,  1.833f,  0.0f,   0.0f, }
 };
 
-static const float smpte240m_full[16] =
+static const vl_csc_matrix smpte240m_full =
 {
-   1.164f,  0.0f,    1.794f, 0.0f,
-   1.164f, -0.258f, -0.543f, 0.0f,
-   1.164f,  2.079f,  0.0f,   0.0f,
-   0.0f,    0.0f,    0.0f,   1.0f
+   { 1.164f,  0.0f,    1.794f, 0.0f, },
+   { 1.164f, -0.258f, -0.543f, 0.0f, },
+   { 1.164f,  2.079f,  0.0f,   0.0f, }
 };
 
-static const float identity[16] =
+static const vl_csc_matrix identity =
 {
-   1.0f, 0.0f, 0.0f, 0.0f,
-   0.0f, 1.0f, 0.0f, 0.0f,
-   0.0f, 0.0f, 1.0f, 0.0f,
-   0.0f, 0.0f, 0.0f, 1.0f
+   { 1.0f, 0.0f, 0.0f, 0.0f, },
+   { 0.0f, 1.0f, 0.0f, 0.0f, },
+   { 0.0f, 0.0f, 1.0f, 0.0f, }
 };
 
 const struct vl_procamp vl_default_procamp = {
@@ -181,7 +174,7 @@ const struct vl_procamp vl_default_procamp = {
 void vl_csc_get_matrix(enum VL_CSC_COLOR_STANDARD cs,
                        struct vl_procamp *procamp,
                        bool full_range,
-                       float *matrix)
+                       vl_csc_matrix *matrix)
 {
    float ybias = full_range ? -16.0f/255.0f : 0.0f;
    float cbbias = -128.0f/255.0f;
@@ -193,44 +186,45 @@ void vl_csc_get_matrix(enum VL_CSC_COLOR_STANDARD cs,
    float b = p->brightness;
    float h = p->hue;
 
-   const float *cstd;
+   const vl_csc_matrix *cstd;
 
    assert(matrix);
 
    switch (cs) {
       case VL_CSC_COLOR_STANDARD_BT_601:
-         cstd = full_range ? &bt_601_full[0] : &bt_601[0];
+         cstd = full_range ? &bt_601_full : &bt_601;
          break;
       case VL_CSC_COLOR_STANDARD_BT_709:
-         cstd = full_range ? &bt_709_full[0] : &bt_709[0];
+         cstd = full_range ? &bt_709_full : &bt_709;
          break;
       case VL_CSC_COLOR_STANDARD_SMPTE_240M:
-         cstd = full_range ? &smpte240m_full[0] : &smpte240m[0];
+         cstd = full_range ? &smpte240m_full : &smpte240m;
          break;
       case VL_CSC_COLOR_STANDARD_IDENTITY:
       default:
          assert(cs == VL_CSC_COLOR_STANDARD_IDENTITY);
-         memcpy(matrix, &identity[0], sizeof(float) * 16);
+         memcpy(matrix, identity, sizeof(vl_csc_matrix));
          return;
    }
 
-   matrix[ 0] = c*cstd[ 0];
-   matrix[ 1] = c*cstd[ 1]*s*cosf(h) - c*cstd[ 2]*s*sinf(h);
-   matrix[ 2] = c*cstd[ 2]*s*cosf(h) + c*cstd[ 1]*s*sinf(h);
-   matrix[ 3] = cstd[ 3] + cstd[ 0]*(b + c*ybias) + cstd[ 1]*(c*cbbias*s*cosf(h) + c*crbias*s*sinf(h)) + cstd[ 2]*(c*crbias*s*cosf(h) - c*cbbias*s*sinf(h));
-
-   matrix[ 4] = c*cstd[ 4];
-   matrix[ 5] = c*cstd[ 5]*s*cosf(h) - c*cstd[ 6]*s*sinf(h);
-   matrix[ 6] = c*cstd[ 6]*s*cosf(h) + c*cstd[ 5]*s*sinf(h);
-   matrix[ 7] = cstd[ 7] + cstd[ 4]*(b + c*ybias) + cstd[ 5]*(c*cbbias*s*cosf(h) + c*crbias*s*sinf(h)) + cstd[ 6]*(c*crbias*s*cosf(h) - c*cbbias*s*sinf(h));
-
-   matrix[ 8] = c*cstd[ 8];
-   matrix[ 9] = c*cstd[ 9]*s*cosf(h) - c*cstd[10]*s*sinf(h);
-   matrix[10] = c*cstd[10]*s*cosf(h) + c*cstd[ 9]*s*sinf(h);
-   matrix[11] = cstd[11] + cstd[ 8]*(b + c*ybias) + cstd[ 9]*(c*cbbias*s*cosf(h) + c*crbias*s*sinf(h)) + cstd[10]*(c*crbias*s*cosf(h) - c*cbbias*s*sinf(h));
-
-   matrix[12] = c*cstd[12];
-   matrix[13] = c*cstd[13]*s*cos(h) - c*cstd[14]*s*sin(h);
-   matrix[14] = c*cstd[14]*s*cos(h) + c*cstd[13]*s*sin(h);
-   matrix[15] = cstd[15] + cstd[12]*(b + c*ybias) + cstd[13]*(c*cbbias*s*cos(h) + c*crbias*s*sin(h)) + cstd[14]*(c*crbias*s*cos(h) - c*cbbias*s*sin(h));
+   (*matrix)[0][0] = c * (*cstd)[0][0];
+   (*matrix)[0][1] = c * (*cstd)[0][1] * s * cosf(h) - c * (*cstd)[0][2] * s * sinf(h);
+   (*matrix)[0][2] = c * (*cstd)[0][2] * s * cosf(h) + c * (*cstd)[0][1] * s * sinf(h);
+   (*matrix)[0][3] = (*cstd)[0][3] + (*cstd)[0][0] * (b + c * ybias) +
+                     (*cstd)[0][1] * (c * cbbias * s * cosf(h) + c * crbias * s * sinf(h)) +
+                     (*cstd)[0][2] * (c * crbias * s * cosf(h) - c * cbbias * s * sinf(h));
+
+   (*matrix)[1][0] = c * (*cstd)[1][0];
+   (*matrix)[1][1] = c * (*cstd)[1][1] * s * cosf(h) - c * (*cstd)[1][2] * s * sinf(h);
+   (*matrix)[1][2] = c * (*cstd)[1][2] * s * cosf(h) + c * (*cstd)[1][1] * s * sinf(h);
+   (*matrix)[1][3] = (*cstd)[1][3] + (*cstd)[1][0] * (b + c * ybias) +
+                     (*cstd)[1][1] * (c * cbbias * s * cosf(h) + c * crbias * s * sinf(h)) +
+                     (*cstd)[1][2] * (c * crbias * s * cosf(h) - c * cbbias * s * sinf(h));
+
+   (*matrix)[2][0] = c * (*cstd)[2][0];
+   (*matrix)[2][1] = c * (*cstd)[2][1] * s * cosf(h) - c * (*cstd)[2][2] * s * sinf(h);
+   (*matrix)[2][2] = c * (*cstd)[2][2] * s * cosf(h) + c * (*cstd)[2][1] * s * sinf(h);
+   (*matrix)[2][3] = (*cstd)[2][3] + (*cstd)[2][0] * (b + c * ybias) +
+                     (*cstd)[2][1] * (c * cbbias * s * cosf(h) + c * crbias * s * sinf(h)) +
+                     (*cstd)[2][2] * (c * crbias * s * cosf(h) - c * cbbias * s * sinf(h));
 }
diff --git a/src/gallium/auxiliary/vl/vl_csc.h b/src/gallium/auxiliary/vl/vl_csc.h
index 7a31dd4eb26..4e36e0b0eda 100644
--- a/src/gallium/auxiliary/vl/vl_csc.h
+++ b/src/gallium/auxiliary/vl/vl_csc.h
@@ -30,6 +30,8 @@
 
 #include "pipe/p_compiler.h"
 
+typedef float vl_csc_matrix[3][4];
+
 struct vl_procamp
 {
    float brightness;
@@ -51,6 +53,6 @@ extern const struct vl_procamp vl_default_procamp;
 void vl_csc_get_matrix(enum VL_CSC_COLOR_STANDARD cs,
                        struct vl_procamp *procamp,
                        bool full_range,
-                       float *matrix);
+                       vl_csc_matrix *matrix);
 
 #endif /* vl_csc_h */
diff --git a/src/gallium/state_trackers/vdpau/mixer.c b/src/gallium/state_trackers/vdpau/mixer.c
index 9a97501e456..d3b8946eb8e 100644
--- a/src/gallium/state_trackers/vdpau/mixer.c
+++ b/src/gallium/state_trackers/vdpau/mixer.c
@@ -64,9 +64,9 @@ vlVdpVideoMixerCreate(VdpDevice device,
    vmixer->device = dev;
    vl_compositor_init_state(&vmixer->cstate, dev->context);
 
-   vl_csc_get_matrix(VL_CSC_COLOR_STANDARD_BT_601, NULL, true, vmixer->csc);
+   vl_csc_get_matrix(VL_CSC_COLOR_STANDARD_BT_601, NULL, true, &vmixer->csc);
    if (!debug_get_bool_option("G3DVL_NO_CSC", FALSE))
-      vl_compositor_set_csc_matrix(&vmixer->cstate, vmixer->csc);
+      vl_compositor_set_csc_matrix(&vmixer->cstate, (const vl_csc_matrix *)&vmixer->csc);
 
    *mixer = vlAddDataHTAB(vmixer);
    if (*mixer == 0) {
@@ -553,11 +553,11 @@ vlVdpVideoMixerSetAttributeValues(VdpVideoMixer mixer,
          vdp_csc = attribute_values[i];
          vmixer->custom_csc = !!vdp_csc;
          if (!vdp_csc)
-            vl_csc_get_matrix(VL_CSC_COLOR_STANDARD_BT_601, NULL, 1, vmixer->csc);
+            vl_csc_get_matrix(VL_CSC_COLOR_STANDARD_BT_601, NULL, 1, &vmixer->csc);
          else
-            memcpy(vmixer->csc, vdp_csc, sizeof(float)*12);
+            memcpy(vmixer->csc, vdp_csc, sizeof(vl_csc_matrix));
          if (!debug_get_bool_option("G3DVL_NO_CSC", FALSE))
-            vl_compositor_set_csc_matrix(&vmixer->cstate, vmixer->csc);
+            vl_compositor_set_csc_matrix(&vmixer->cstate, (const vl_csc_matrix *)&vmixer->csc);
          break;
 
       case VDP_VIDEO_MIXER_ATTRIBUTE_NOISE_REDUCTION_LEVEL:
@@ -709,7 +709,6 @@ vlVdpGenerateCSCMatrix(VdpProcamp *procamp,
                        VdpColorStandard standard,
                        VdpCSCMatrix *csc_matrix)
 {
-   float matrix[16];
    enum VL_CSC_COLOR_STANDARD vl_std;
    struct vl_procamp camp;
 
@@ -729,7 +728,6 @@ vlVdpGenerateCSCMatrix(VdpProcamp *procamp,
    camp.contrast = procamp->contrast;
    camp.saturation = procamp->saturation;
    camp.hue = procamp->hue;
-   vl_csc_get_matrix(vl_std, &camp, 1, matrix);
-   memcpy(csc_matrix, matrix, sizeof(float)*12);
+   vl_csc_get_matrix(vl_std, &camp, true, csc_matrix);
    return VDP_STATUS_OK;
 }
diff --git a/src/gallium/state_trackers/vdpau/vdpau_private.h b/src/gallium/state_trackers/vdpau/vdpau_private.h
index 1b617847d4a..1d776ab398e 100644
--- a/src/gallium/state_trackers/vdpau/vdpau_private.h
+++ b/src/gallium/state_trackers/vdpau/vdpau_private.h
@@ -38,7 +38,9 @@
 
 #include "util/u_debug.h"
 #include "util/u_rect.h"
+
 #include "vl/vl_compositor.h"
+#include "vl/vl_csc.h"
 #include "vl/vl_matrix_filter.h"
 #include "vl/vl_median_filter.h"
 
@@ -322,9 +324,11 @@ typedef struct
 
    unsigned video_width, video_height;
    enum pipe_video_chroma_format chroma_format;
-   unsigned max_layers, skip_chroma_deint, custom_csc;
+   unsigned max_layers, skip_chroma_deint;
    float luma_key_min, luma_key_max;
-   float csc[16];
+
+   bool custom_csc;
+   vl_csc_matrix csc;
 } vlVdpVideoMixer;
 
 typedef struct
diff --git a/src/gallium/state_trackers/xvmc/attributes.c b/src/gallium/state_trackers/xvmc/attributes.c
index da7b6493ba8..58bafb1343f 100644
--- a/src/gallium/state_trackers/xvmc/attributes.c
+++ b/src/gallium/state_trackers/xvmc/attributes.c
@@ -77,7 +77,7 @@ Status XvMCSetAttribute(Display *dpy, XvMCContext *context, Atom attribute, int
 {
    XvMCContextPrivate *context_priv;
    const char *attr;
-   float csc[16];
+   vl_csc_matrix csc;
 
    assert(dpy);
 
@@ -108,9 +108,9 @@ Status XvMCSetAttribute(Display *dpy, XvMCContext *context, Atom attribute, int
    vl_csc_get_matrix
    (
       context_priv->color_standard,
-      &context_priv->procamp, true, csc
+      &context_priv->procamp, true, &csc
    );
-   vl_compositor_set_csc_matrix(&context_priv->cstate, csc);
+   vl_compositor_set_csc_matrix(&context_priv->cstate, (const vl_csc_matrix *)&csc);
 
    XVMC_MSG(XVMC_TRACE, "[XvMC] Set attribute %s to value %d.\n", attr, value);
 
diff --git a/src/gallium/state_trackers/xvmc/context.c b/src/gallium/state_trackers/xvmc/context.c
index c598cad7455..7b97a67d662 100644
--- a/src/gallium/state_trackers/xvmc/context.c
+++ b/src/gallium/state_trackers/xvmc/context.c
@@ -192,7 +192,7 @@ Status XvMCCreateContext(Display *dpy, XvPortID port, int surface_type_id,
    struct vl_screen *vscreen;
    struct pipe_context *pipe;
    XvMCContextPrivate *context_priv;
-   float csc[16];
+   vl_csc_matrix csc;
 
    XVMC_MSG(XVMC_TRACE, "[XvMC] Creating context %p.\n", context);
 
@@ -289,9 +289,9 @@ Status XvMCCreateContext(Display *dpy, XvPortID port, int surface_type_id,
    vl_csc_get_matrix
    (
       context_priv->color_standard,
-      &context_priv->procamp, true, csc
+      &context_priv->procamp, true, &csc
    );
-   vl_compositor_set_csc_matrix(&context_priv->cstate, csc);
+   vl_compositor_set_csc_matrix(&context_priv->cstate, (const vl_csc_matrix *)&csc);
 
    context_priv->vscreen = vscreen;
    context_priv->pipe = pipe;