glsl/linker: eliminate unused and set-but-unused built-in varyings

This eliminates built-in varyings such as gl_Color, gl_SecondaryColor,
gl_TexCoord, and gl_FogFragCoord if they are unused by the next stage or
not written at all (e.g. gl_TexCoord elements). The gl_TexCoord array is
broken down into separate vec4s if needed.

v2: - use a switch statement in varying_info_visitor::visit(ir_variable*)
    - use snprintf
    - disable the optimization for GLES2

Reviewed-by: Ian Romanick <[email protected]>

5 files changed, 496 insertions, 2 deletions

diff --git a/src/glsl/Makefile.sources b/src/glsl/Makefile.sources
index acd19d1ff3d..979c4165f50 100644
--- a/src/glsl/Makefile.sources
+++ b/src/glsl/Makefile.sources
@@ -82,6 +82,7 @@ LIBGLSL_FILES = \
 	$(GLSL_SRCDIR)/opt_constant_variable.cpp \
 	$(GLSL_SRCDIR)/opt_copy_propagation.cpp \
 	$(GLSL_SRCDIR)/opt_copy_propagation_elements.cpp \
+	$(GLSL_SRCDIR)/opt_dead_builtin_varyings.cpp \
 	$(GLSL_SRCDIR)/opt_dead_code.cpp \
 	$(GLSL_SRCDIR)/opt_dead_code_local.cpp \
 	$(GLSL_SRCDIR)/opt_dead_functions.cpp \
diff --git a/src/glsl/ir_optimization.h b/src/glsl/ir_optimization.h
index d38d5e30365..fad6f1bfe5f 100644
--- a/src/glsl/ir_optimization.h
+++ b/src/glsl/ir_optimization.h
@@ -76,6 +76,10 @@ bool do_constant_variable_unlinked(exec_list *instructions);
 bool do_copy_propagation(exec_list *instructions);
 bool do_copy_propagation_elements(exec_list *instructions);
 bool do_constant_propagation(exec_list *instructions);
+void do_dead_builtin_varyings(struct gl_context *ctx,
+                              exec_list *producer, exec_list *consumer,
+                              unsigned num_tfeedback_decls,
+                              class tfeedback_decl *tfeedback_decls);
 bool do_dead_code(exec_list *instructions, bool uniform_locations_assigned);
 bool do_dead_code_local(exec_list *instructions);
 bool do_dead_code_unlinked(exec_list *instructions);
diff --git a/src/glsl/link_varyings.h b/src/glsl/link_varyings.h
index daa9d79c94f..7f7be353b6d 100644
--- a/src/glsl/link_varyings.h
+++ b/src/glsl/link_varyings.h
@@ -125,6 +125,10 @@ public:
          return this->vector_elements * this->matrix_columns * this->size;
    }
 
+   unsigned get_location() const {
+      return this->location;
+   }
+
 private:
    /**
     * The name that was supplied to glTransformFeedbackVaryings.  Used for
diff --git a/src/glsl/linker.cpp b/src/glsl/linker.cpp
index 6d7357820c9..ba97ade25e0 100644
--- a/src/glsl/linker.cpp
+++ b/src/glsl/linker.cpp
@@ -1887,6 +1887,9 @@ link_shaders(struct gl_context *ctx, struct gl_shader_program *prog)
             goto done;
       }
 
+      do_dead_builtin_varyings(ctx, sh->ir, NULL,
+                               num_tfeedback_decls, tfeedback_decls);
+
       demote_shader_inputs_and_outputs(sh, ir_var_shader_out);
 
       /* Eliminate code that is now dead due to unused outputs being demoted.
@@ -1895,11 +1898,13 @@ link_shaders(struct gl_context *ctx, struct gl_shader_program *prog)
          ;
    }
    else if (first == MESA_SHADER_FRAGMENT) {
-      /* If the program only contains a fragment shader, just demote
-       * user-defined varyings.
+      /* If the program only contains a fragment shader...
        */
       gl_shader *const sh = prog->_LinkedShaders[first];
 
+      do_dead_builtin_varyings(ctx, NULL, sh->ir,
+                               num_tfeedback_decls, tfeedback_decls);
+
       demote_shader_inputs_and_outputs(sh, ir_var_shader_in);
 
       while (do_dead_code(sh->ir, false))
@@ -1919,6 +1924,10 @@ link_shaders(struct gl_context *ctx, struct gl_shader_program *prog)
                 tfeedback_decls))
          goto done;
 
+      do_dead_builtin_varyings(ctx, sh_i->ir, sh_next->ir,
+                next == MESA_SHADER_FRAGMENT ? num_tfeedback_decls : 0,
+                tfeedback_decls);
+
       demote_shader_inputs_and_outputs(sh_i, ir_var_shader_out);
       demote_shader_inputs_and_outputs(sh_next, ir_var_shader_in);
 
diff --git a/src/glsl/opt_dead_builtin_varyings.cpp b/src/glsl/opt_dead_builtin_varyings.cpp
new file mode 100644
index 00000000000..71bb7c41c73
--- /dev/null
+++ b/src/glsl/opt_dead_builtin_varyings.cpp
@@ -0,0 +1,476 @@
+/*
+ * Copyright © 2013 Marek Olšák <[email protected]>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * \file opt_dead_builtin_varyings.cpp
+ *
+ * This eliminates the built-in shader outputs which are either not written
+ * at all or not used by the next stage. It also eliminates unused elements
+ * of gl_TexCoord inputs, which reduces the overall varying usage.
+ * The varyings handled here are the primary and secondary color, the fog,
+ * and the texture coordinates (gl_TexCoord).
+ *
+ * This pass is necessary, because the Mesa GLSL linker cannot eliminate
+ * built-in varyings like it eliminates user-defined varyings, because
+ * the built-in varyings have pre-assigned locations. Also, the elimination
+ * of unused gl_TexCoord elements requires its own lowering pass anyway.
+ *
+ * It's implemented by replacing all occurences of dead varyings with
+ * temporary variables, which creates dead code. It is recommended to run
+ * a dead-code elimination pass after this.
+ *
+ * If any texture coordinate slots can be eliminated, the gl_TexCoord array is
+ * broken down into separate vec4 variables with locations equal to
+ * VARYING_SLOT_TEX0 + i.
+ */
+
+#include "ir.h"
+#include "ir_rvalue_visitor.h"
+#include "ir_optimization.h"
+#include "ir_print_visitor.h"
+#include "glsl_types.h"
+#include "link_varyings.h"
+
+
+/**
+ * This obtains detailed information about built-in varyings from shader code.
+ */
+class varying_info_visitor : public ir_hierarchical_visitor {
+public:
+   /* "mode" can be either ir_var_shader_in or ir_var_shader_out */
+   varying_info_visitor(ir_variable_mode mode)
+      : lower_texcoord_array(true),
+        texcoord_array(NULL),
+        texcoord_usage(0),
+        color_usage(0),
+        tfeedback_color_usage(0),
+        fog(NULL),
+        has_fog(false),
+        tfeedback_has_fog(false),
+        mode(mode)
+   {
+      memset(color, 0, sizeof(color));
+      memset(backcolor, 0, sizeof(backcolor));
+   }
+
+   virtual ir_visitor_status visit_enter(ir_dereference_array *ir)
+   {
+      ir_variable *var = ir->variable_referenced();
+
+      if (var && var->mode == this->mode &&
+          var->location == VARYING_SLOT_TEX0) {
+         this->texcoord_array = var;
+
+         ir_constant *index = ir->array_index->as_constant();
+         if (index == NULL) {
+            /* There is variable indexing, we can't lower the texcoord array.
+             */
+            this->texcoord_usage |= (1 << var->type->array_size()) - 1;
+            this->lower_texcoord_array = false;
+         }
+         else {
+            this->texcoord_usage |= 1 << index->get_uint_component(0);
+         }
+
+         /* Don't visit the leaves of ir_dereference_array. */
+         return visit_continue_with_parent;
+      }
+
+      return visit_continue;
+   }
+
+   virtual ir_visitor_status visit(ir_dereference_variable *ir)
+   {
+      ir_variable *var = ir->variable_referenced();
+
+      if (var->mode == this->mode && var->type->is_array() &&
+          var->location == VARYING_SLOT_TEX0) {
+         /* This is a whole array dereference like "gl_TexCoord = x;",
+          * there's probably no point in lowering that.
+          */
+         this->texcoord_usage |= (1 << var->type->array_size()) - 1;
+         this->lower_texcoord_array = false;
+      }
+      return visit_continue;
+   }
+
+   virtual ir_visitor_status visit(ir_variable *var)
+   {
+      if (var->mode != this->mode)
+         return visit_continue;
+
+      /* Handle colors and fog. */
+      switch (var->location) {
+      case VARYING_SLOT_COL0:
+         this->color[0] = var;
+         this->color_usage |= 1;
+         break;
+      case VARYING_SLOT_COL1:
+         this->color[1] = var;
+         this->color_usage |= 2;
+         break;
+      case VARYING_SLOT_BFC0:
+         this->backcolor[0] = var;
+         this->color_usage |= 1;
+         break;
+      case VARYING_SLOT_BFC1:
+         this->backcolor[1] = var;
+         this->color_usage |= 2;
+         break;
+      case VARYING_SLOT_FOGC:
+         this->fog = var;
+         this->has_fog = true;
+         break;
+      }
+
+      return visit_continue;
+   }
+
+   void get(exec_list *ir,
+            unsigned num_tfeedback_decls,
+            tfeedback_decl *tfeedback_decls)
+   {
+      /* Handle the transform feedback varyings. */
+      for (unsigned i = 0; i < num_tfeedback_decls; i++) {
+         if (!tfeedback_decls[i].is_varying())
+            continue;
+
+         unsigned location = tfeedback_decls[i].get_location();
+
+         switch (location) {
+         case VARYING_SLOT_COL0:
+         case VARYING_SLOT_BFC0:
+            this->tfeedback_color_usage |= 1;
+            break;
+         case VARYING_SLOT_COL1:
+         case VARYING_SLOT_BFC1:
+            this->tfeedback_color_usage |= 2;
+            break;
+         case VARYING_SLOT_FOGC:
+            this->tfeedback_has_fog = true;
+            break;
+         default:
+            if (location >= VARYING_SLOT_TEX0 &&
+                location <= VARYING_SLOT_TEX7) {
+               this->lower_texcoord_array = false;
+            }
+         }
+      }
+
+      /* Process the shader. */
+      visit_list_elements(this, ir);
+
+      if (!this->texcoord_array) {
+         this->lower_texcoord_array = false;
+      }
+   }
+
+   bool lower_texcoord_array;
+   ir_variable *texcoord_array;
+   unsigned texcoord_usage; /* bitmask */
+
+   ir_variable *color[2];
+   ir_variable *backcolor[2];
+   unsigned color_usage; /* bitmask */
+   unsigned tfeedback_color_usage; /* bitmask */
+
+   ir_variable *fog;
+   bool has_fog;
+   bool tfeedback_has_fog;
+
+   ir_variable_mode mode;
+};
+
+
+/**
+ * This replaces unused varyings with temporary variables.
+ *
+ * If "ir" is the producer, the "external" usage should come from
+ * the consumer. It also works the other way around. If either one is
+ * missing, set the "external" usage to a full mask.
+ */
+class replace_varyings_visitor : public ir_rvalue_visitor {
+public:
+   replace_varyings_visitor(exec_list *ir,
+                            const varying_info_visitor *info,
+                            unsigned external_texcoord_usage,
+                            unsigned external_color_usage,
+                            bool external_has_fog)
+      : info(info), new_fog(NULL)
+   {
+      void *const ctx = ir;
+
+      memset(this->new_texcoord, 0, sizeof(this->new_texcoord));
+      memset(this->new_color, 0, sizeof(this->new_color));
+      memset(this->new_backcolor, 0, sizeof(this->new_backcolor));
+
+      const char *mode_str =
+         info->mode == ir_var_shader_in ? "in" : "out";
+
+      /* Handle texcoord outputs.
+       *
+       * We're going to break down the gl_TexCoord array into separate
+       * variables. First, add declarations of the new variables all
+       * occurences of gl_TexCoord will be replaced with.
+       */
+      if (info->lower_texcoord_array) {
+         for (int i = MAX_TEXTURE_COORD_UNITS-1; i >= 0; i--) {
+            if (info->texcoord_usage & (1 << i)) {
+               char name[32];
+
+               if (!(external_texcoord_usage & (1 << i))) {
+                  /* This varying is unused in the next stage. Declare
+                   * a temporary instead of an output. */
+                  snprintf(name, 32, "gl_%s_TexCoord%i_dummy", mode_str, i);
+                  this->new_texcoord[i] =
+                     new (ctx) ir_variable(glsl_type::vec4_type, name,
+                                           ir_var_temporary);
+               }
+               else {
+                  snprintf(name, 32, "gl_%s_TexCoord%i", mode_str, i);
+                  this->new_texcoord[i] =
+                     new(ctx) ir_variable(glsl_type::vec4_type, name,
+                                          info->mode);
+                  this->new_texcoord[i]->location = VARYING_SLOT_TEX0 + i;
+                  this->new_texcoord[i]->explicit_location = true;
+                  this->new_texcoord[i]->explicit_index = 0;
+               }
+
+               ir->head->insert_before(new_texcoord[i]);
+            }
+         }
+      }
+
+      /* Create dummy variables which will replace set-but-unused color and
+       * fog outputs.
+       */
+      external_color_usage |= info->tfeedback_color_usage;
+
+      for (int i = 0; i < 2; i++) {
+         char name[32];
+
+         if (!(external_color_usage & (1 << i))) {
+            if (info->color[i]) {
+               snprintf(name, 32, "gl_%s_FrontColor%i_dummy", mode_str, i);
+               this->new_color[i] =
+                  new (ctx) ir_variable(glsl_type::vec4_type, name,
+                                        ir_var_temporary);
+            }
+
+            if (info->backcolor[i]) {
+               snprintf(name, 32, "gl_%s_BackColor%i_dummy", mode_str, i);
+               this->new_backcolor[i] =
+                  new (ctx) ir_variable(glsl_type::vec4_type, name,
+                                        ir_var_temporary);
+            }
+         }
+      }
+
+      if (!external_has_fog && !info->tfeedback_has_fog &&
+          info->fog) {
+         char name[32];
+
+         snprintf(name, 32, "gl_%s_FogFragCoord_dummy", mode_str);
+         this->new_fog = new (ctx) ir_variable(glsl_type::float_type, name,
+                                               ir_var_temporary);
+      }
+
+      /* Now do the replacing. */
+      visit_list_elements(this, ir);
+   }
+
+   virtual ir_visitor_status visit(ir_variable *var)
+   {
+      /* Remove the gl_TexCoord array. */
+      if (this->info->lower_texcoord_array &&
+          var == this->info->texcoord_array) {
+         var->remove();
+      }
+
+      /* Replace set-but-unused color and fog outputs with dummy variables. */
+      for (int i = 0; i < 2; i++) {
+         if (var == this->info->color[i] && this->new_color[i]) {
+            var->replace_with(this->new_color[i]);
+         }
+         if (var == this->info->backcolor[i] &&
+             this->new_backcolor[i]) {
+            var->replace_with(this->new_backcolor[i]);
+         }
+      }
+
+      if (var == this->info->fog && this->new_fog) {
+         var->replace_with(this->new_fog);
+      }
+
+      return visit_continue;
+   }
+
+   virtual void handle_rvalue(ir_rvalue **rvalue)
+   {
+      if (!*rvalue)
+         return;
+
+      void *ctx = ralloc_parent(*rvalue);
+
+      /* Replace an array dereference gl_TexCoord[i] with a single
+       * variable dereference representing gl_TexCoord[i].
+       */
+      if (this->info->lower_texcoord_array) {
+         /* gl_TexCoord[i] occurence */
+         ir_dereference_array *const da = (*rvalue)->as_dereference_array();
+
+         if (da && da->variable_referenced() ==
+             this->info->texcoord_array) {
+            unsigned i = da->array_index->as_constant()->get_uint_component(0);
+
+            *rvalue = new(ctx) ir_dereference_variable(this->new_texcoord[i]);
+            return;
+         }
+      }
+
+      /* Replace set-but-unused color and fog outputs with dummy variables. */
+      ir_dereference_variable *const dv = (*rvalue)->as_dereference_variable();
+      if (!dv)
+         return;
+
+      ir_variable *var = dv->variable_referenced();
+
+      for (int i = 0; i < 2; i++) {
+         if (var == this->info->color[i] && this->new_color[i]) {
+            *rvalue = new(ctx) ir_dereference_variable(this->new_color[i]);
+            return;
+         }
+         if (var == this->info->backcolor[i] &&
+             this->new_backcolor[i]) {
+            *rvalue = new(ctx) ir_dereference_variable(this->new_backcolor[i]);
+            return;
+         }
+      }
+
+      if (var == this->info->fog && this->new_fog) {
+         *rvalue = new(ctx) ir_dereference_variable(this->new_fog);
+      }
+   }
+
+   virtual ir_visitor_status visit_leave(ir_assignment *ir)
+   {
+      handle_rvalue(&ir->rhs);
+      handle_rvalue(&ir->condition);
+
+      /* We have to use set_lhs when changing the LHS of an assignment. */
+      ir_rvalue *lhs = ir->lhs;
+
+      handle_rvalue(&lhs);
+      if (lhs != ir->lhs) {
+         ir->set_lhs(lhs);
+      }
+
+      return visit_continue;
+   }
+
+private:
+   const varying_info_visitor *info;
+   struct ir_variable *new_texcoord[MAX_TEXTURE_COORD_UNITS];
+   struct ir_variable *new_color[2];
+   struct ir_variable *new_backcolor[2];
+   struct ir_variable *new_fog;
+};
+
+
+static void
+lower_texcoord_array(exec_list *ir, const varying_info_visitor *info)
+{
+   replace_varyings_visitor(ir, info,
+                            (1 << MAX_TEXTURE_COORD_UNITS) - 1,
+                            1 | 2, true);
+}
+
+
+void
+do_dead_builtin_varyings(struct gl_context *ctx,
+                         exec_list *producer, exec_list *consumer,
+                         unsigned num_tfeedback_decls,
+                         tfeedback_decl *tfeedback_decls)
+{
+   /* This optimization has no effect with the core context and GLES2, because
+    * the built-in varyings we're eliminating here are not available there.
+    *
+    * EXT_separate_shader_objects doesn't allow this optimization,
+    * because a program object can be bound partially (e.g. only one
+    * stage of a program object can be bound).
+    */
+   if (ctx->API == API_OPENGL_CORE ||
+       ctx->API == API_OPENGLES2 ||
+       ctx->Extensions.EXT_separate_shader_objects) {
+      return;
+   }
+
+   /* Information about built-in varyings. */
+   varying_info_visitor producer_info(ir_var_shader_out);
+   varying_info_visitor consumer_info(ir_var_shader_in);
+
+   if (producer) {
+      producer_info.get(producer, num_tfeedback_decls, tfeedback_decls);
+
+      if (!consumer) {
+         /* At least eliminate unused gl_TexCoord elements. */
+         if (producer_info.lower_texcoord_array) {
+            lower_texcoord_array(producer, &producer_info);
+         }
+         return;
+      }
+   }
+
+   if (consumer) {
+      consumer_info.get(consumer, 0, NULL);
+
+      if (!producer) {
+         /* At least eliminate unused gl_TexCoord elements. */
+         if (consumer_info.lower_texcoord_array) {
+            lower_texcoord_array(consumer, &consumer_info);
+         }
+         return;
+      }
+   }
+
+   /* Eliminate the varyings unused by the other shader. */
+   if (producer_info.lower_texcoord_array ||
+       producer_info.color_usage ||
+       producer_info.has_fog) {
+      replace_varyings_visitor(producer,
+                               &producer_info,
+                               consumer_info.texcoord_usage,
+                               consumer_info.color_usage,
+                               consumer_info.has_fog);
+   }
+
+   if (consumer_info.lower_texcoord_array ||
+       consumer_info.color_usage ||
+       consumer_info.has_fog) {
+      replace_varyings_visitor(consumer,
+                               &consumer_info,
+                               producer_info.texcoord_usage,
+                               producer_info.color_usage,
+                               producer_info.has_fog);
+   }
+}