mesa/st: add support for NIR as possible driver IR

Signed-off-by: Rob Clark <[email protected]>
Acked-by: Eric Anholt <[email protected]>

8 files changed, 632 insertions, 12 deletions

diff --git a/src/mesa/Makefile.sources b/src/mesa/Makefile.sources
index 81928c8cdf4..840a68e1db6 100644
--- a/src/mesa/Makefile.sources
+++ b/src/mesa/Makefile.sources
@@ -489,6 +489,7 @@ STATETRACKER_FILES = \
 	state_tracker/st_gen_mipmap.c \
 	state_tracker/st_gen_mipmap.h \
 	state_tracker/st_gl_api.h \
+	state_tracker/st_glsl_to_nir.cpp \
 	state_tracker/st_glsl_to_tgsi.cpp \
 	state_tracker/st_glsl_to_tgsi.h \
 	state_tracker/st_glsl_types.cpp \
diff --git a/src/mesa/SConscript b/src/mesa/SConscript
index 71741010217..434800e4560 100644
--- a/src/mesa/SConscript
+++ b/src/mesa/SConscript
@@ -13,6 +13,7 @@ env = env.Clone()
 env.MSVC2013Compat()
 
 env.Append(CPPPATH = [
+    '../compiler/nir',  # for generated nir_opcodes.h, etc
     '#/src',
     '#/src/mapi',
     '#/src/glsl',
diff --git a/src/mesa/state_tracker/st_glsl_to_nir.cpp b/src/mesa/state_tracker/st_glsl_to_nir.cpp
new file mode 100644
index 00000000000..6cfbb8e7683
--- /dev/null
+++ b/src/mesa/state_tracker/st_glsl_to_nir.cpp
@@ -0,0 +1,442 @@
+/*
+ * Copyright © 2015 Red Hat
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "st_nir.h"
+
+#include "pipe/p_defines.h"
+#include "pipe/p_screen.h"
+#include "pipe/p_context.h"
+
+#include "program/program.h"
+#include "program/prog_statevars.h"
+#include "program/prog_parameter.h"
+#include "program/ir_to_mesa.h"
+#include "program/hash_table.h"
+#include "main/mtypes.h"
+#include "main/errors.h"
+#include "main/shaderapi.h"
+#include "main/uniforms.h"
+
+#include "st_context.h"
+#include "st_program.h"
+#include "st_glsl_types.h"
+
+#include "compiler/nir/nir.h"
+#include "compiler/nir/glsl_to_nir.h"
+#include "compiler/glsl_types.h"
+#include "compiler/glsl/ir.h"
+
+
+/* Depending on PIPE_CAP_TGSI_TEXCOORD (st->needs_texcoord_semantic) we
+ * may need to fix up varying slots so the glsl->nir path is aligned
+ * with the anything->tgsi->nir path.
+ */
+static void
+st_nir_fixup_varying_slots(struct st_context *st, struct exec_list *var_list)
+{
+   if (st->needs_texcoord_semantic)
+      return;
+
+   nir_foreach_variable(var, var_list) {
+      if (var->data.location >= VARYING_SLOT_VAR0) {
+         var->data.location += 9;
+      } else if ((var->data.location >= VARYING_SLOT_TEX0) &&
+               (var->data.location <= VARYING_SLOT_TEX7)) {
+         var->data.location += VARYING_SLOT_VAR0 - VARYING_SLOT_TEX0;
+      }
+   }
+}
+
+/* input location assignment for VS inputs must be handled specially, so
+ * that it is aligned w/ st's vbo state.
+ * (This isn't the case with, for ex, FS inputs, which only need to agree
+ * on varying-slot w/ the VS outputs)
+ */
+static void
+st_nir_assign_vs_in_locations(struct gl_program *prog,
+                              struct exec_list *var_list, unsigned *size)
+{
+   unsigned attr, num_inputs = 0;
+   unsigned input_to_index[VERT_ATTRIB_MAX] = {0};
+
+   /* TODO de-duplicate w/ similar code in st_translate_vertex_program()? */
+   for (attr = 0; attr < VERT_ATTRIB_MAX; attr++) {
+      if ((prog->InputsRead & BITFIELD64_BIT(attr)) != 0) {
+         input_to_index[attr] = num_inputs;
+         num_inputs++;
+         if ((prog->DoubleInputsRead & BITFIELD64_BIT(attr)) != 0) {
+            /* add placeholder for second part of a double attribute */
+            num_inputs++;
+         }
+      }
+   }
+
+   *size = 0;
+   nir_foreach_variable(var, var_list) {
+      attr = var->data.location;
+      assert(attr < ARRAY_SIZE(input_to_index));
+      var->data.driver_location = input_to_index[attr];
+      (*size)++;
+   }
+}
+
+static int
+st_nir_lookup_parameter_index(const struct gl_program_parameter_list *params,
+                              const char *name)
+{
+   int loc = _mesa_lookup_parameter_index(params, name);
+
+   /* is there a better way to do this?  If we have something like:
+    *
+    *    struct S {
+    *           float f;
+    *           vec4 v;
+    *    };
+    *    uniform S color;
+    *
+    * Then what we get in prog->Parameters looks like:
+    *
+    *    0: Name=color.f, Type=6, DataType=1406, Size=1
+    *    1: Name=color.v, Type=6, DataType=8b52, Size=4
+    *
+    * So the name doesn't match up and _mesa_lookup_parameter_index()
+    * fails.  In this case just find the first matching "color.*"..
+    *
+    * Note for arrays you could end up w/ color[n].f, for example.
+    *
+    * glsl_to_tgsi works slightly differently in this regard.  It is
+    * emitting something more low level, so it just translates the
+    * params list 1:1 to CONST[] regs.  Going from GLSL IR to TGSI,
+    * it just calculates the additional offset of struct field members
+    * in glsl_to_tgsi_visitor::visit(ir_dereference_record *ir) or
+    * glsl_to_tgsi_visitor::visit(ir_dereference_array *ir).  It never
+    * needs to work backwards to get base var loc from the param-list
+    * which already has them separated out.
+    */
+   if (loc < 0) {
+      int namelen = strlen(name);
+      for (unsigned i = 0; i < params->NumParameters; i++) {
+         struct gl_program_parameter *p = &params->Parameters[i];
+         if ((strncmp(p->Name, name, namelen) == 0) &&
+             ((p->Name[namelen] == '.') || (p->Name[namelen] == '['))) {
+            loc = i;
+            break;
+         }
+      }
+   }
+
+   return loc;
+}
+
+static void
+st_nir_assign_uniform_locations(struct gl_program *prog,
+                                struct gl_shader_program *shader_program,
+                                struct exec_list *uniform_list, unsigned *size)
+{
+   int max = 0;
+   int shaderidx = 0;
+
+   nir_foreach_variable(uniform, uniform_list) {
+      int loc;
+
+      /*
+       * UBO's have their own address spaces, so don't count them towards the
+       * number of global uniforms
+       */
+      if ((uniform->data.mode == nir_var_uniform || uniform->data.mode == nir_var_shader_storage) &&
+          uniform->interface_type != NULL)
+         continue;
+
+      if (uniform->type->is_sampler()) {
+         unsigned val;
+         bool found = shader_program->UniformHash->get(val, uniform->name);
+         loc = shaderidx++;
+         assert(found);
+         /* this ensure that nir_lower_samplers looks at the correct
+          * shader_program->UniformStorage[location]:
+          */
+         uniform->data.location = val;
+      } else if (strncmp(uniform->name, "gl_", 3) == 0) {
+         const gl_state_index *const stateTokens = (gl_state_index *)uniform->state_slots[0].tokens;
+         /* This state reference has already been setup by ir_to_mesa, but we'll
+          * get the same index back here.
+          */
+         loc = _mesa_add_state_reference(prog->Parameters, stateTokens);
+      } else {
+         loc = st_nir_lookup_parameter_index(prog->Parameters, uniform->name);
+      }
+
+      uniform->data.driver_location = loc;
+
+      max = MAX2(max, loc + st_glsl_type_size(uniform->type));
+   }
+   *size = max;
+}
+
+extern "C" {
+
+/* First half of converting glsl_to_nir.. this leaves things in a pre-
+ * nir_lower_io state, so that shader variants can more easily insert/
+ * replace variables, etc.
+ */
+nir_shader *
+st_glsl_to_nir(struct st_context *st, struct gl_program *prog,
+               struct gl_shader_program *shader_program,
+               gl_shader_stage stage)
+{
+   struct pipe_screen *pscreen = st->pipe->screen;
+   unsigned ptarget = st_shader_stage_to_ptarget(stage);
+   const nir_shader_compiler_options *options;
+   nir_shader *nir;
+
+   assert(pscreen->get_compiler_options);   /* drivers using NIR must implement this */
+
+   options = (const nir_shader_compiler_options *)
+      pscreen->get_compiler_options(pscreen, PIPE_SHADER_IR_NIR, ptarget);
+   assert(options);
+
+   if (prog->nir)
+      return prog->nir;
+
+   nir = glsl_to_nir(shader_program, stage, options);
+   prog->nir = nir;
+
+   NIR_PASS_V(nir, nir_lower_io_to_temporaries,
+         nir_shader_get_entrypoint(nir),
+         true, true);
+   NIR_PASS_V(nir, nir_lower_global_vars_to_local);
+   NIR_PASS_V(nir, nir_split_var_copies);
+   NIR_PASS_V(nir, nir_lower_var_copies);
+   NIR_PASS_V(nir, st_nir_lower_builtin);
+
+   /* fragment shaders may need : */
+   if (stage == MESA_SHADER_FRAGMENT) {
+      static const gl_state_index wposTransformState[STATE_LENGTH] = {
+         STATE_INTERNAL, STATE_FB_WPOS_Y_TRANSFORM
+      };
+      nir_lower_wpos_ytransform_options wpos_options = {0};
+      struct pipe_screen *pscreen = st->pipe->screen;
+
+      memcpy(wpos_options.state_tokens, wposTransformState,
+             sizeof(wpos_options.state_tokens));
+      wpos_options.fs_coord_origin_upper_left =
+         pscreen->get_param(pscreen, PIPE_CAP_TGSI_FS_COORD_ORIGIN_UPPER_LEFT);
+      wpos_options.fs_coord_origin_lower_left =
+         pscreen->get_param(pscreen, PIPE_CAP_TGSI_FS_COORD_ORIGIN_LOWER_LEFT);
+      wpos_options.fs_coord_pixel_center_integer =
+         pscreen->get_param(pscreen, PIPE_CAP_TGSI_FS_COORD_PIXEL_CENTER_INTEGER);
+      wpos_options.fs_coord_pixel_center_half_integer =
+         pscreen->get_param(pscreen, PIPE_CAP_TGSI_FS_COORD_PIXEL_CENTER_HALF_INTEGER);
+
+      if (nir_lower_wpos_ytransform(nir, &wpos_options)) {
+         nir_validate_shader(nir);
+         _mesa_add_state_reference(prog->Parameters, wposTransformState);
+      }
+   }
+
+   if (st->ctx->_Shader->Flags & GLSL_DUMP) {
+      _mesa_log("\n");
+      _mesa_log("NIR IR for linked %s program %d:\n",
+             _mesa_shader_stage_to_string(stage),
+             shader_program->Name);
+      nir_print_shader(nir, _mesa_get_log_file());
+      _mesa_log("\n\n");
+   }
+
+   return nir;
+}
+
+/* TODO any better helper somewhere to sort a list? */
+
+static void
+insert_sorted(struct exec_list *var_list, nir_variable *new_var)
+{
+   nir_foreach_variable(var, var_list) {
+      if (var->data.location > new_var->data.location) {
+         exec_node_insert_node_before(&var->node, &new_var->node);
+         return;
+      }
+   }
+   exec_list_push_tail(var_list, &new_var->node);
+}
+
+static void
+sort_varyings(struct exec_list *var_list)
+{
+   struct exec_list new_list;
+   exec_list_make_empty(&new_list);
+   nir_foreach_variable_safe(var, var_list) {
+      exec_node_remove(&var->node);
+      insert_sorted(&new_list, var);
+   }
+   exec_list_move_nodes_to(&new_list, var_list);
+}
+
+/* Second half of preparing nir from glsl, which happens after shader
+ * variant lowering.
+ */
+void
+st_finalize_nir(struct st_context *st, struct gl_program *prog, nir_shader *nir)
+{
+   NIR_PASS_V(nir, nir_split_var_copies);
+   NIR_PASS_V(nir, nir_lower_var_copies);
+   NIR_PASS_V(nir, nir_lower_io_types);
+
+   if (nir->stage == MESA_SHADER_VERTEX) {
+      /* Needs special handling so drvloc matches the vbo state: */
+      st_nir_assign_vs_in_locations(prog, &nir->inputs, &nir->num_inputs);
+      sort_varyings(&nir->outputs);
+      nir_assign_var_locations(&nir->outputs,
+                               &nir->num_outputs,
+                               st_glsl_type_size);
+      st_nir_fixup_varying_slots(st, &nir->outputs);
+   } else if (nir->stage == MESA_SHADER_FRAGMENT) {
+      sort_varyings(&nir->inputs);
+      nir_assign_var_locations(&nir->inputs,
+                               &nir->num_inputs,
+                               st_glsl_type_size);
+      st_nir_fixup_varying_slots(st, &nir->inputs);
+      nir_assign_var_locations(&nir->outputs,
+                               &nir->num_outputs,
+                               st_glsl_type_size);
+   } else {
+      unreachable("invalid shader type for tgsi bypass\n");
+   }
+
+   struct gl_shader_program *shader_program;
+   switch (nir->stage) {
+   case MESA_SHADER_VERTEX:
+      shader_program = ((struct st_vertex_program *)prog)->shader_program;
+      break;
+   case MESA_SHADER_FRAGMENT:
+      shader_program = ((struct st_fragment_program *)prog)->shader_program;
+      break;
+   default:
+      assert(!"should not be reached");
+      return;
+   }
+
+   st_nir_assign_uniform_locations(prog, shader_program,
+                                   &nir->uniforms, &nir->num_uniforms);
+
+   NIR_PASS_V(nir, nir_lower_system_values);
+   NIR_PASS_V(nir, nir_lower_io, nir_var_all, st_glsl_type_size);
+   NIR_PASS_V(nir, nir_lower_samplers, shader_program);
+}
+
+struct gl_program *
+st_nir_get_mesa_program(struct gl_context *ctx,
+                        struct gl_shader_program *shader_program,
+                        struct gl_shader *shader)
+{
+   struct gl_program *prog;
+   GLenum target = _mesa_shader_stage_to_program(shader->Stage);
+
+   validate_ir_tree(shader->ir);
+
+   prog = ctx->Driver.NewProgram(ctx, target, shader_program->Name);
+   if (!prog)
+      return NULL;
+
+   prog->Parameters = _mesa_new_parameter_list();
+
+   _mesa_copy_linked_program_data(shader->Stage, shader_program, prog);
+   _mesa_generate_parameters_list_for_uniforms(shader_program, shader,
+                                               prog->Parameters);
+
+   /* Make a pass over the IR to add state references for any built-in
+    * uniforms that are used.  This has to be done now (during linking).
+    * Code generation doesn't happen until the first time this shader is
+    * used for rendering.  Waiting until then to generate the parameters is
+    * too late.  At that point, the values for the built-in uniforms won't
+    * get sent to the shader.
+    */
+   foreach_in_list(ir_instruction, node, shader->ir) {
+      ir_variable *var = node->as_variable();
+
+      if ((var == NULL) || (var->data.mode != ir_var_uniform) ||
+          (strncmp(var->name, "gl_", 3) != 0))
+         continue;
+
+      const ir_state_slot *const slots = var->get_state_slots();
+      assert(slots != NULL);
+
+      for (unsigned int i = 0; i < var->get_num_state_slots(); i++) {
+         _mesa_add_state_reference(prog->Parameters,
+                                   (gl_state_index *) slots[i].tokens);
+      }
+   }
+
+   if (ctx->_Shader->Flags & GLSL_DUMP) {
+      _mesa_log("\n");
+      _mesa_log("GLSL IR for linked %s program %d:\n",
+             _mesa_shader_stage_to_string(shader->Stage),
+             shader_program->Name);
+      _mesa_print_ir(_mesa_get_log_file(), shader->ir, NULL);
+      _mesa_log("\n\n");
+   }
+
+   prog->Instructions = NULL;
+   prog->NumInstructions = 0;
+
+   do_set_program_inouts(shader->ir, prog, shader->Stage);
+
+   prog->SamplersUsed = shader->active_samplers;
+   prog->ShadowSamplers = shader->shadow_samplers;
+   _mesa_update_shader_textures_used(shader_program, prog);
+
+   _mesa_reference_program(ctx, &shader->Program, prog);
+
+   /* Avoid reallocation of the program parameter list, because the uniform
+    * storage is only associated with the original parameter list.
+    * This should be enough for Bitmap and DrawPixels constants.
+    */
+   _mesa_reserve_parameter_storage(prog->Parameters, 8);
+
+   /* This has to be done last.  Any operation the can cause
+    * prog->ParameterValues to get reallocated (e.g., anything that adds a
+    * program constant) has to happen before creating this linkage.
+    */
+   _mesa_associate_uniform_storage(ctx, shader_program, prog->Parameters);
+
+   struct st_vertex_program *stvp;
+   struct st_fragment_program *stfp;
+
+   switch (shader->Type) {
+   case GL_VERTEX_SHADER:
+      stvp = (struct st_vertex_program *)prog;
+      stvp->shader_program = shader_program;
+      break;
+   case GL_FRAGMENT_SHADER:
+      stfp = (struct st_fragment_program *)prog;
+      stfp->shader_program = shader_program;
+      break;
+   default:
+      assert(!"should not be reached");
+      return NULL;
+   }
+
+   return prog;
+}
+
+} /* extern "C" */
diff --git a/src/mesa/state_tracker/st_glsl_to_tgsi.cpp b/src/mesa/state_tracker/st_glsl_to_tgsi.cpp
index d8a0cb249aa..9d2057e6383 100644
--- a/src/mesa/state_tracker/st_glsl_to_tgsi.cpp
+++ b/src/mesa/state_tracker/st_glsl_to_tgsi.cpp
@@ -53,6 +53,7 @@
 #include "st_mesa_to_tgsi.h"
 #include "st_format.h"
 #include "st_glsl_types.h"
+#include "st_nir.h"
 
 
 #define PROGRAM_ANY_CONST ((1 << PROGRAM_STATE_VAR) |    \
@@ -6437,9 +6438,9 @@ out:
  * generating Mesa IR.
  */
 static struct gl_program *
-get_mesa_program(struct gl_context *ctx,
-                 struct gl_shader_program *shader_program,
-                 struct gl_shader *shader)
+get_mesa_program_tgsi(struct gl_context *ctx,
+                      struct gl_shader_program *shader_program,
+                      struct gl_shader *shader)
 {
    glsl_to_tgsi_visitor* v;
    struct gl_program *prog;
@@ -6647,6 +6648,29 @@ get_mesa_program(struct gl_context *ctx,
    return prog;
 }
 
+static struct gl_program *
+get_mesa_program(struct gl_context *ctx,
+                 struct gl_shader_program *shader_program,
+                 struct gl_shader *shader)
+{
+   struct pipe_screen *pscreen = ctx->st->pipe->screen;
+   unsigned ptarget = st_shader_stage_to_ptarget(shader->Stage);
+   enum pipe_shader_ir preferred_ir = (enum pipe_shader_ir)
+      pscreen->get_shader_param(pscreen, ptarget, PIPE_SHADER_CAP_PREFERRED_IR);
+   if (preferred_ir == PIPE_SHADER_IR_NIR) {
+      /* TODO only for GLSL VS/FS for now: */
+      switch (shader->Type) {
+      case GL_VERTEX_SHADER:
+      case GL_FRAGMENT_SHADER:
+         return st_nir_get_mesa_program(ctx, shader_program, shader);
+      default:
+         break;
+      }
+   }
+   return get_mesa_program_tgsi(ctx, shader_program, shader);
+}
+
+
 extern "C" {
 
 static void
@@ -6849,9 +6873,17 @@ st_translate_stream_output_info(glsl_to_tgsi_visitor *glsl_to_tgsi,
                                 const GLuint outputMapping[],
                                 struct pipe_stream_output_info *so)
 {
-   unsigned i;
    struct gl_transform_feedback_info *info =
       &glsl_to_tgsi->shader_program->LinkedTransformFeedback;
+   st_translate_stream_output_info2(info, outputMapping, so);
+}
+
+void
+st_translate_stream_output_info2(struct gl_transform_feedback_info *info,
+                                const GLuint outputMapping[],
+                                struct pipe_stream_output_info *so)
+{
+   unsigned i;
 
    for (i = 0; i < info->NumOutputs; i++) {
       so->output[i].register_index =
diff --git a/src/mesa/state_tracker/st_glsl_to_tgsi.h b/src/mesa/state_tracker/st_glsl_to_tgsi.h
index 774588a111b..0f485fb2541 100644
--- a/src/mesa/state_tracker/st_glsl_to_tgsi.h
+++ b/src/mesa/state_tracker/st_glsl_to_tgsi.h
@@ -63,6 +63,11 @@ st_translate_stream_output_info(struct glsl_to_tgsi_visitor *glsl_to_tgsi,
                                 const GLuint outputMapping[],
                                 struct pipe_stream_output_info *so);
 
+void
+st_translate_stream_output_info2(struct gl_transform_feedback_info *info,
+                                const GLuint outputMapping[],
+                                struct pipe_stream_output_info *so);
+
 unsigned
 _mesa_sysval_to_semantic(unsigned sysval);
 
diff --git a/src/mesa/state_tracker/st_nir.h b/src/mesa/state_tracker/st_nir.h
index 1192981bda7..49ba573df2d 100644
--- a/src/mesa/state_tracker/st_nir.h
+++ b/src/mesa/state_tracker/st_nir.h
@@ -24,8 +24,29 @@
 #ifndef ST_NIR_H
 #define ST_NIR_H
 
+#include "st_context.h"
+#include "compiler/shader_enums.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 typedef struct nir_shader nir_shader;
 
 void st_nir_lower_builtin(nir_shader *shader);
+nir_shader * st_glsl_to_nir(struct st_context *st, struct gl_program *prog,
+                            struct gl_shader_program *shader_program,
+                            gl_shader_stage stage);
+
+void st_finalize_nir(struct st_context *st, struct gl_program *prog, nir_shader *nir);
+
+struct gl_program *
+st_nir_get_mesa_program(struct gl_context *ctx,
+                        struct gl_shader_program *shader_program,
+                        struct gl_shader *shader);
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif /* ST_NIR_H */
diff --git a/src/mesa/state_tracker/st_program.c b/src/mesa/state_tracker/st_program.c
index 252e4400087..f2b55379923 100644
--- a/src/mesa/state_tracker/st_program.c
+++ b/src/mesa/state_tracker/st_program.c
@@ -38,6 +38,8 @@
 #include "program/prog_print.h"
 #include "program/programopt.h"
 
+#include "compiler/nir/nir.h"
+
 #include "pipe/p_context.h"
 #include "pipe/p_defines.h"
 #include "pipe/p_shader_tokens.h"
@@ -54,6 +56,7 @@
 #include "st_program.h"
 #include "st_mesa_to_tgsi.h"
 #include "st_atifs_to_tgsi.h"
+#include "st_nir.h"
 #include "cso_cache/cso_context.h"
 
 
@@ -70,10 +73,10 @@ delete_vp_variant(struct st_context *st, struct st_vp_variant *vpv)
       
    if (vpv->draw_shader)
       draw_delete_vertex_shader( st->draw, vpv->draw_shader );
-      
-   if (vpv->tgsi.tokens)
+
+   if (((vpv->tgsi.type == PIPE_SHADER_IR_TGSI)) && vpv->tgsi.tokens)
       ureg_free_tokens(vpv->tgsi.tokens);
-      
+
    free( vpv );
 }
 
@@ -96,7 +99,7 @@ st_release_vp_variants( struct st_context *st,
 
    stvp->variants = NULL;
 
-   if (stvp->tgsi.tokens) {
+   if ((stvp->tgsi.type == PIPE_SHADER_IR_TGSI) && stvp->tgsi.tokens) {
       tgsi_free_tokens(stvp->tgsi.tokens);
       stvp->tgsi.tokens = NULL;
    }
@@ -133,7 +136,7 @@ st_release_fp_variants(struct st_context *st, struct st_fragment_program *stfp)
 
    stfp->variants = NULL;
 
-   if (stfp->tgsi.tokens) {
+   if ((stfp->tgsi.type == PIPE_SHADER_IR_TGSI) && stfp->tgsi.tokens) {
       ureg_free_tokens(stfp->tgsi.tokens);
       stfp->tgsi.tokens = NULL;
    }
@@ -361,9 +364,23 @@ st_translate_vertex_program(struct st_context *st,
    output_semantic_name[num_outputs] = TGSI_SEMANTIC_EDGEFLAG;
    output_semantic_index[num_outputs] = 0;
 
-   if (!stvp->glsl_to_tgsi)
+   if (!stvp->glsl_to_tgsi && !stvp->shader_program)
       _mesa_remove_output_reads(&stvp->Base.Base, PROGRAM_OUTPUT);
 
+   if (stvp->shader_program) {
+      nir_shader *nir = st_glsl_to_nir(st, &stvp->Base.Base,
+                                       stvp->shader_program,
+                                       MESA_SHADER_VERTEX);
+
+      stvp->tgsi.type = PIPE_SHADER_IR_NIR;
+      stvp->tgsi.ir.nir = nir;
+
+      st_translate_stream_output_info2(&stvp->shader_program->LinkedTransformFeedback,
+                                       stvp->result_to_output,
+                                       &stvp->tgsi.stream_output);
+      return true;
+   }
+
    ureg = ureg_create_with_screen(PIPE_SHADER_VERTEX, st->pipe->screen);
    if (ureg == NULL)
       return false;
@@ -446,10 +463,27 @@ st_create_vp_variant(struct st_context *st,
    struct pipe_context *pipe = st->pipe;
 
    vpv->key = *key;
-   vpv->tgsi.tokens = tgsi_dup_tokens(stvp->tgsi.tokens);
    vpv->tgsi.stream_output = stvp->tgsi.stream_output;
    vpv->num_inputs = stvp->num_inputs;
 
+   if (stvp->tgsi.type == PIPE_SHADER_IR_NIR) {
+      vpv->tgsi.type = PIPE_SHADER_IR_NIR;
+      vpv->tgsi.ir.nir = nir_shader_clone(NULL, stvp->tgsi.ir.nir);
+      if (key->clamp_color)
+         NIR_PASS_V(vpv->tgsi.ir.nir, nir_lower_clamp_color_outputs);
+      if (key->passthrough_edgeflags)
+         NIR_PASS_V(vpv->tgsi.ir.nir, nir_lower_passthrough_edgeflags);
+
+      st_finalize_nir(st, &stvp->Base.Base, vpv->tgsi.ir.nir);
+
+      vpv->driver_shader = pipe->create_vs_state(pipe, &vpv->tgsi);
+      /* driver takes ownership of IR: */
+      vpv->tgsi.ir.nir = NULL;
+      return vpv;
+   }
+
+   vpv->tgsi.tokens = tgsi_dup_tokens(stvp->tgsi.tokens);
+
    /* Emulate features. */
    if (key->clamp_color || key->passthrough_edgeflags) {
       const struct tgsi_token *tokens;
@@ -559,7 +593,7 @@ st_translate_fragment_program(struct st_context *st,
 
    memset(inputSlotToAttr, ~0, sizeof(inputSlotToAttr));
 
-   if (!stfp->glsl_to_tgsi) {
+   if (!stfp->glsl_to_tgsi && !stfp->shader_program) {
       _mesa_remove_output_reads(&stfp->Base.Base, PROGRAM_OUTPUT);
       if (st->ctx->Const.GLSLFragCoordIsSysVal)
          _mesa_program_fragment_position_to_sysval(&stfp->Base.Base);
@@ -767,6 +801,17 @@ st_translate_fragment_program(struct st_context *st,
       }
    }
 
+   if (stfp->shader_program) {
+      nir_shader *nir = st_glsl_to_nir(st, &stfp->Base.Base,
+                                       stfp->shader_program,
+                                       MESA_SHADER_FRAGMENT);
+
+      stfp->tgsi.type = PIPE_SHADER_IR_NIR;
+      stfp->tgsi.ir.nir = nir;
+
+      return true;
+   }
+
    ureg = ureg_create_with_screen(PIPE_SHADER_FRAGMENT, st->pipe->screen);
    if (ureg == NULL)
       return false;
@@ -881,6 +926,73 @@ st_create_fp_variant(struct st_context *st,
    if (!variant)
       return NULL;
 
+   if (stfp->tgsi.type == PIPE_SHADER_IR_NIR) {
+      tgsi.type = PIPE_SHADER_IR_NIR;
+      tgsi.ir.nir = nir_shader_clone(NULL, stfp->tgsi.ir.nir);
+
+      if (key->clamp_color)
+         NIR_PASS_V(tgsi.ir.nir, nir_lower_clamp_color_outputs);
+
+      if (key->persample_shading) {
+          nir_shader *shader = tgsi.ir.nir;
+          nir_foreach_variable(var, &shader->inputs)
+             var->data.sample = true;
+      }
+
+      assert(!(key->bitmap && key->drawpixels));
+
+      /* glBitmap */
+      if (key->bitmap) {
+         nir_lower_bitmap_options options = {0};
+
+         variant->bitmap_sampler = ffs(~stfp->Base.Base.SamplersUsed) - 1;
+         options.sampler = variant->bitmap_sampler;
+         options.swizzle_xxxx = (st->bitmap.tex_format == PIPE_FORMAT_L8_UNORM);
+
+         NIR_PASS_V(tgsi.ir.nir, nir_lower_bitmap, &options);
+      }
+
+      /* glDrawPixels (color only) */
+      if (key->drawpixels) {
+         nir_lower_drawpixels_options options = {0};
+         unsigned samplers_used = stfp->Base.Base.SamplersUsed;
+
+         /* Find the first unused slot. */
+         variant->drawpix_sampler = ffs(~samplers_used) - 1;
+         options.drawpix_sampler = variant->drawpix_sampler;
+         samplers_used |= (1 << variant->drawpix_sampler);
+
+         options.pixel_maps = key->pixelMaps;
+         if (key->pixelMaps) {
+            variant->pixelmap_sampler = ffs(~samplers_used) - 1;
+            options.pixelmap_sampler = variant->pixelmap_sampler;
+         }
+
+         options.scale_and_bias = key->scaleAndBias;
+         if (key->scaleAndBias) {
+            _mesa_add_state_reference(params, scale_state);
+            memcpy(options.scale_state_tokens, scale_state,
+                   sizeof(options.scale_state_tokens));
+            _mesa_add_state_reference(params, bias_state);
+            memcpy(options.bias_state_tokens, bias_state,
+                   sizeof(options.bias_state_tokens));
+         }
+
+         _mesa_add_state_reference(params, texcoord_state);
+         memcpy(options.texcoord_state_tokens, texcoord_state,
+                sizeof(options.texcoord_state_tokens));
+
+         NIR_PASS_V(tgsi.ir.nir, nir_lower_drawpixels, &options);
+      }
+
+      st_finalize_nir(st, &stfp->Base.Base, tgsi.ir.nir);
+
+      variant->driver_shader = pipe->create_fs_state(pipe, &tgsi);
+      variant->key = *key;
+
+      return variant;
+   }
+
    tgsi.tokens = stfp->tgsi.tokens;
 
    assert(!(key->bitmap && key->drawpixels));
diff --git a/src/mesa/state_tracker/st_program.h b/src/mesa/state_tracker/st_program.h
index 7c90fd74e14..e6db1069025 100644
--- a/src/mesa/state_tracker/st_program.h
+++ b/src/mesa/state_tracker/st_program.h
@@ -108,6 +108,9 @@ struct st_fragment_program
    struct glsl_to_tgsi_visitor* glsl_to_tgsi;
    struct ati_fragment_shader *ati_fs;
 
+   /* used when bypassing glsl_to_tgsi: */
+   struct gl_shader_program *shader_program;
+
    struct st_fp_variant *variants;
 };
 
@@ -164,6 +167,9 @@ struct st_vertex_program
    struct pipe_shader_state tgsi;
    struct glsl_to_tgsi_visitor* glsl_to_tgsi;
 
+   /* used when bypassing glsl_to_tgsi: */
+   struct gl_shader_program *shader_program;
+
    /** maps a Mesa VERT_ATTRIB_x to a packed TGSI input index */
    /** maps a TGSI input index back to a Mesa VERT_ATTRIB_x */
    GLuint index_to_input[PIPE_MAX_SHADER_INPUTS];