nv50/ir: import new shader backend code

21 files changed, 13395 insertions, 1 deletions

diff --git a/src/gallium/drivers/nv50/Makefile b/src/gallium/drivers/nv50/Makefile
index 220adf696b3..18e30b0a54f 100644
--- a/src/gallium/drivers/nv50/Makefile
+++ b/src/gallium/drivers/nv50/Makefile
@@ -3,7 +3,7 @@ include $(TOP)/configs/current
 
 LIBNAME = nv50
 
-# get C_SOURCES
+# get C/CPP_SOURCES
 include Makefile.sources
 
 LIBRARY_INCLUDES = \
diff --git a/src/gallium/drivers/nv50/Makefile.sources b/src/gallium/drivers/nv50/Makefile.sources
index 756f90be979..cc9321bef7e 100644
--- a/src/gallium/drivers/nv50/Makefile.sources
+++ b/src/gallium/drivers/nv50/Makefile.sources
@@ -21,3 +21,17 @@ C_SOURCES := \
 	nv50_pc_regalloc.c \
 	nv50_push.c \
 	nv50_query.c
+
+CPP_SOURCES := \
+	codegen/nv50_ir.cpp \
+	codegen/nv50_ir_bb.cpp \
+	codegen/nv50_ir_build_util.cpp \
+	codegen/nv50_ir_emit_nv50.cpp \
+	codegen/nv50_ir_from_tgsi.cpp \
+	codegen/nv50_ir_graph.cpp \
+	codegen/nv50_ir_peephole.cpp \
+	codegen/nv50_ir_print.cpp \
+	codegen/nv50_ir_ra.cpp \
+	codegen/nv50_ir_ssa.cpp \
+	codegen/nv50_ir_target.cpp \
+	codegen/nv50_ir_util.cpp
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir.cpp b/src/gallium/drivers/nv50/codegen/nv50_ir.cpp
new file mode 100644
index 00000000000..ff2e6ef3401
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir.cpp
@@ -0,0 +1,1008 @@
+
+#include "nv50_ir.h"
+#include "nv50_ir_target.h"
+#include "nv50_ir_driver.h"
+
+extern "C" {
+#include "nv50/nv50_program.h"
+#include "nv50/nv50_debug.h"
+}
+
+namespace nv50_ir {
+
+Modifier::Modifier(operation op)
+{
+   switch (op) {
+   case OP_NEG: bits = NV50_IR_MOD_NEG; break;
+   case OP_ABS: bits = NV50_IR_MOD_ABS; break;
+   case OP_SAT: bits = NV50_IR_MOD_SAT; break;
+   case OP_NOP: bits = NV50_IR_MOD_NOT; break;
+   default:
+      bits = 0;
+      break;
+   }
+}
+
+Modifier Modifier::operator*(const Modifier m) const
+{
+   unsigned int a, b, c;
+
+   b = m.bits;
+   if (this->bits & NV50_IR_MOD_ABS)
+      b &= ~NV50_IR_MOD_NEG;
+
+   a = (this->bits ^ b)      & (NV50_IR_MOD_NOT | NV50_IR_MOD_NEG);
+   c = (this->bits | m.bits) & (NV50_IR_MOD_ABS | NV50_IR_MOD_SAT);
+
+   return Modifier(a | c);
+}
+
+ValueRef::ValueRef() : value(0), insn(0), next(this), prev(this)
+{
+   indirect[0] = -1;
+   indirect[1] = -1;
+   usedAsPtr = false;
+}
+
+ValueRef::~ValueRef()
+{
+   this->set(NULL);
+}
+
+ImmediateValue *ValueRef::getImmediate() const
+{
+   Value *src = value;
+
+   while (src) {
+      if (src->reg.file == FILE_IMMEDIATE)
+         return src->asImm();
+
+      Instruction *insn = src->getUniqueInsn();
+
+      src = (insn && insn->op == OP_MOV) ? insn->getSrc(0) : NULL;
+   }
+   return NULL;
+}
+
+ValueDef::ValueDef() : value(0), insn(0), next(this), prev(this)
+{
+   // nothing to do
+}
+
+ValueDef::~ValueDef()
+{
+   this->set(NULL);
+}
+
+void
+ValueRef::set(const ValueRef &ref)
+{
+   this->set(ref.get());
+   mod = ref.mod;
+   indirect[0] = ref.indirect[0];
+   indirect[1] = ref.indirect[1];
+}
+
+void
+ValueRef::set(Value *refVal)
+{
+   if (value == refVal)
+      return;
+   if (value) {
+      if (value->uses == this)
+         value->uses = (next == this) ? NULL : next;
+      value->unref();
+      DLLIST_DEL(this);
+   }
+
+   if (refVal) {
+      if (refVal->uses)
+         DLLIST_ADDTAIL(refVal->uses, this);
+      else
+         refVal->uses = this;
+      refVal->ref();
+   }
+   value = refVal;
+}
+
+void
+ValueDef::set(Value *defVal)
+{
+   assert(next != this || prev == this); // check that SSA hack isn't active
+
+   if (value == defVal)
+      return;
+   if (value) {
+      if (value->defs == this)
+         value->defs = (next == this) ? NULL : next;
+      DLLIST_DEL(this);
+   }
+
+   if (defVal) {
+      if (defVal->defs)
+         DLLIST_ADDTAIL(defVal->defs, this);
+      else
+         defVal->defs = this;
+   }
+   value = defVal;
+}
+
+// TODO: make me faster by using a safe iterator
+void
+ValueDef::replace(Value *repVal, bool doSet)
+{
+   ValueRef **refs = new ValueRef * [value->refCount()];
+   int n = 0;
+
+   if (!refs && value->refCount())
+      FATAL("memory allocation failed");
+
+   for (ValueRef::Iterator iter = value->uses->iterator(); !iter.end();
+        iter.next()) {
+      assert(n < value->refCount());
+      refs[n++] = iter.get();
+   }
+   while (n)
+      refs[--n]->set(repVal);
+
+   if (doSet)
+      this->set(repVal);
+
+   if (refs)
+      delete[] refs;
+}
+
+void
+ValueDef::mergeDefs(ValueDef *join)
+{
+   DLLIST_MERGE(this, join, ValueDef *);
+}
+
+Value::Value()
+{
+  refCnt = 0;
+  uses = NULL;
+  defs = NULL;
+  join = this;
+
+  memset(&reg, 0, sizeof(reg));
+  reg.size = 4;
+}
+
+bool
+Value::coalesce(Value *jval, bool force)
+{
+   Value *repr = this->join; // new representative
+   Value *jrep = jval->join;
+
+   if (reg.file != jval->reg.file || reg.size != jval->reg.size) {
+      if (!force)
+         return false;
+      ERROR("forced coalescing of values of different sizes/files");
+   }
+
+   if (!force && (repr->reg.data.id != jrep->reg.data.id)) {
+      if (repr->reg.data.id >= 0 &&
+          jrep->reg.data.id >= 0)
+            return false;
+      if (jrep->reg.data.id >= 0) {
+         repr = jval->join;
+         jrep = this->join;
+         jval = this;
+      }
+
+      // need to check all fixed register values of the program for overlap
+      Function *func = defs->getInsn()->bb->getFunction();
+
+      // TODO: put values in by register-id bins per function
+      ArrayList::Iterator iter = func->allLValues.iterator();
+      for (; !iter.end(); iter.next()) {
+         Value *fixed = reinterpret_cast<Value *>(iter.get());
+         assert(fixed);
+         if (fixed->reg.data.id == repr->reg.data.id)
+            if (fixed->livei.overlaps(jrep->livei))
+               return false;
+      }
+   }
+   if (repr->livei.overlaps(jrep->livei)) {
+      if (!force)
+         return false;
+      // do we really want this ? if at all, only for constraint ops
+      INFO("NOTE: forced coalescing with live range overlap\n");
+   }
+
+   ValueDef::Iterator iter = jrep->defs->iterator();
+   for (; !iter.end(); iter.next())
+      iter.get()->get()->join = repr;
+
+   repr->defs->mergeDefs(jrep->defs);
+   repr->livei.unify(jrep->livei);
+
+   assert(repr->join == repr && jval->join == repr);
+   return true;
+}
+
+LValue::LValue(Function *fn, DataFile file)
+{
+   reg.file = file;
+   reg.size = (file != FILE_PREDICATE) ? 4 : 1;
+   reg.data.id = -1;
+
+   affinity = -1;
+
+   fn->add(this, this->id);
+}
+
+LValue::LValue(Function *fn, LValue *lval)
+{
+   assert(lval);
+
+   reg.file = lval->reg.file;
+   reg.size = lval->reg.size;
+   reg.data.id = -1;
+
+   affinity = -1;
+
+   fn->add(this, this->id);
+}
+
+Value *LValue::clone(Function *func) const
+{
+   LValue *that = new_LValue(func, reg.file);
+
+   that->reg.size = this->reg.size;
+   that->reg.type = this->reg.type;
+   that->reg.data = this->reg.data;
+
+   return that;
+}
+
+Symbol::Symbol(Program *prog, DataFile f, ubyte fidx)
+{
+   baseSym = NULL;
+
+   reg.file = f;
+   reg.fileIndex = fidx;
+   reg.data.offset = 0;
+
+   prog->add(this, this->id);
+}
+
+Value *
+Symbol::clone(Function *func) const
+{
+   Program *prog = func->getProgram();
+
+   Symbol *that = new_Symbol(prog, reg.file, reg.fileIndex);
+
+   that->reg.size = this->reg.size;
+   that->reg.type = this->reg.type;
+   that->reg.data = this->reg.data;
+
+   that->baseSym = this->baseSym;
+
+   return that;
+}
+
+ImmediateValue::ImmediateValue(Program *prog, uint32_t uval)
+{
+   memset(&reg, 0, sizeof(reg));
+
+   reg.file = FILE_IMMEDIATE;
+   reg.size = 4;
+   reg.type = TYPE_U32;
+
+   reg.data.u32 = uval;
+
+   prog->add(this, this->id);
+}
+
+ImmediateValue::ImmediateValue(Program *prog, float fval)
+{
+   memset(&reg, 0, sizeof(reg));
+
+   reg.file = FILE_IMMEDIATE;
+   reg.size = 4;
+   reg.type = TYPE_F32;
+
+   reg.data.f32 = fval;
+
+   prog->add(this, this->id);
+}
+
+ImmediateValue::ImmediateValue(Program *prog, double dval)
+{
+   memset(&reg, 0, sizeof(reg));
+
+   reg.file = FILE_IMMEDIATE;
+   reg.size = 8;
+   reg.type = TYPE_F64;
+
+   reg.data.f64 = dval;
+
+   prog->add(this, this->id);
+}
+
+ImmediateValue::ImmediateValue(const ImmediateValue *proto, DataType ty)
+{
+   reg = proto->reg;
+
+   reg.type = ty;
+   reg.size = typeSizeof(ty);
+}
+
+bool
+ImmediateValue::isInteger(const int i) const
+{
+   switch (reg.type) {
+   case TYPE_S8:
+      return reg.data.s8 == i;
+   case TYPE_U8:
+      return reg.data.u8 == i;
+   case TYPE_S16:
+      return reg.data.s16 == i;
+   case TYPE_U16:
+      return reg.data.u16 == i;
+   case TYPE_S32:
+   case TYPE_U32:
+      return reg.data.s32 == i; // as if ...
+   case TYPE_F32:
+      return reg.data.f32 == static_cast<float>(i);
+   case TYPE_F64:
+      return reg.data.f64 == static_cast<double>(i);
+   default:
+      return false;
+   }
+}
+
+bool
+ImmediateValue::isNegative() const
+{
+   switch (reg.type) {
+   case TYPE_S8:  return reg.data.s8 < 0;
+   case TYPE_S16: return reg.data.s16 < 0;
+   case TYPE_S32:
+   case TYPE_U32: return reg.data.s32 < 0;
+   case TYPE_F32: return reg.data.u32 & (1 << 31);
+   case TYPE_F64: return reg.data.u64 & (1ULL << 63);
+   default:
+      return false;
+   }
+}
+
+bool
+ImmediateValue::isPow2() const
+{
+   switch (reg.type) {
+   case TYPE_U8:
+   case TYPE_U16:
+   case TYPE_U32: return util_is_power_of_two(reg.data.u32);
+   default:
+      return false;
+   }
+}
+
+void
+ImmediateValue::applyLog2()
+{
+   switch (reg.type) {
+   case TYPE_S8:
+   case TYPE_S16:
+   case TYPE_S32:
+      assert(!this->isNegative());
+      // fall through
+   case TYPE_U8:
+   case TYPE_U16:
+   case TYPE_U32:
+      reg.data.u32 = util_logbase2(reg.data.u32);
+      break;
+   case TYPE_F32:
+      reg.data.f32 = log2f(reg.data.f32);
+      break;
+   case TYPE_F64:
+      reg.data.f64 = log2(reg.data.f64);
+      break;
+   default:
+      assert(0);
+      break;
+   }
+}
+
+bool
+ImmediateValue::compare(CondCode cc, float fval) const
+{
+   if (reg.type != TYPE_F32)
+      ERROR("immediate value is not of type f32");
+
+   switch (static_cast<CondCode>(cc & 7)) {
+   case CC_TR: return true;
+   case CC_FL: return false;
+   case CC_LT: return reg.data.f32 <  fval;
+   case CC_LE: return reg.data.f32 <= fval;
+   case CC_GT: return reg.data.f32 >  fval;
+   case CC_GE: return reg.data.f32 >= fval;
+   case CC_EQ: return reg.data.f32 == fval;
+   case CC_NE: return reg.data.f32 != fval;
+   default:
+      assert(0);
+      return false;
+   }
+}
+
+bool
+Value::interfers(const Value *that) const
+{
+   uint32_t idA, idB;
+
+   if (that->reg.file != reg.file || that->reg.fileIndex != reg.fileIndex)
+      return false;
+   if (this->asImm())
+      return false;
+
+   if (this->asSym()) {
+      idA = this->join->reg.data.offset;
+      idB = that->join->reg.data.offset;
+   } else {
+      idA = this->join->reg.data.id * this->reg.size;
+      idB = that->join->reg.data.id * that->reg.size;
+   }
+
+   if (idA < idB)
+      return (idA + this->reg.size > idB);
+   else
+   if (idA > idB)
+      return (idB + that->reg.size > idA);
+   else
+      return (idA == idB);
+}
+
+bool
+Value::equals(const Value *that, bool strict) const
+{
+   that = that->join;
+
+   if (strict)
+      return this == that;
+
+   if (that->reg.file != reg.file || that->reg.fileIndex != reg.fileIndex)
+      return false;
+   if (that->reg.size != this->reg.size)
+      return false;
+
+   if (that->reg.data.id != this->reg.data.id)
+      return false;
+
+   return true;
+}
+
+bool
+ImmediateValue::equals(const Value *that, bool strict) const
+{
+   const ImmediateValue *imm = that->asImm();
+   if (!imm)
+      return false;
+   return reg.data.u64 == imm->reg.data.u64;
+}
+
+bool
+Symbol::equals(const Value *that, bool strict) const
+{
+   if (this->reg.file != that->reg.file)
+      return false;
+   assert(that->asSym());
+
+   if (this->baseSym != that->asSym()->baseSym)
+      return false;
+
+   return this->reg.data.offset == that->reg.data.offset;
+}
+
+void Instruction::init()
+{
+   next = prev = 0;
+
+   cc = CC_ALWAYS;
+   rnd = ROUND_N;
+   cache = CACHE_CA;
+   subOp = 0;
+
+   saturate = 0;
+   join = terminator = 0;
+   ftz = dnz = 0;
+   atomic = 0;
+   perPatch = 0;
+   fixed = 0;
+   encSize = 0;
+   ipa = 0;
+
+   lanes = 0xf;
+
+   postFactor = 0;
+
+   for (int p = 0; p < NV50_IR_MAX_DEFS; ++p)
+      def[p].setInsn(this);
+   for (int p = 0; p < NV50_IR_MAX_SRCS; ++p)
+      src[p].setInsn(this);
+
+   predSrc = -1;
+   flagsDef = -1;
+   flagsSrc = -1;
+}
+
+Instruction::Instruction()
+{
+   init();
+
+   op = OP_NOP;
+   dType = sType = TYPE_F32;
+
+   id = -1;
+   bb = 0;
+}
+
+Instruction::Instruction(Function *fn, operation opr, DataType ty)
+{
+   init();
+
+   op = opr;
+   dType = sType = ty;
+
+   fn->add(this, id);
+}
+
+Instruction::~Instruction()
+{
+   if (bb) {
+      Function *fn = bb->getFunction();
+      bb->remove(this);
+      fn->allInsns.remove(id);
+   }
+
+   for (int s = 0; srcExists(s); ++s)
+      setSrc(s, NULL);
+   // must unlink defs too since the list pointers will get deallocated
+   for (int d = 0; defExists(d); ++d)
+      setDef(d, NULL);
+}
+
+void
+Instruction::setSrc(int s, ValueRef& ref)
+{
+   setSrc(s, ref.get());
+   src[s].mod = ref.mod;
+}
+
+void
+Instruction::swapSources(int a, int b)
+{
+   Value *value = src[a].get();
+   Modifier m = src[a].mod;
+
+   setSrc(a, src[b]);
+
+   src[b].set(value);
+   src[b].mod = m;
+}
+
+void
+Instruction::takeExtraSources(int s, Value *values[3])
+{
+   values[0] = getIndirect(s, 0);
+   if (values[0])
+      setIndirect(s, 0, NULL);
+
+   values[1] = getIndirect(s, 1);
+   if (values[1])
+      setIndirect(s, 1, NULL);
+
+   values[2] = getPredicate();
+   if (values[2])
+      setPredicate(cc, NULL);
+}
+
+void
+Instruction::putExtraSources(int s, Value *values[3])
+{
+   if (values[0])
+      setIndirect(s, 0, values[0]);
+   if (values[1])
+      setIndirect(s, 1, values[1]);
+   if (values[2])
+      setPredicate(cc, values[2]);
+}
+
+Instruction *
+Instruction::clone(bool deep) const
+{
+   Instruction *insn = new_Instruction(bb->getFunction(), op, dType);
+   assert(!asCmp() && !asFlow());
+   cloneBase(insn, deep);
+   return insn;
+}
+
+void
+Instruction::cloneBase(Instruction *insn, bool deep) const
+{
+   insn->sType = this->sType;
+
+   insn->cc = this->cc;
+   insn->rnd = this->rnd;
+   insn->cache = this->cache;
+   insn->subOp = this->subOp;
+
+   insn->saturate = this->saturate;
+   insn->atomic = this->atomic;
+   insn->ftz = this->ftz;
+   insn->dnz = this->dnz;
+   insn->ipa = this->ipa;
+   insn->lanes = this->lanes;
+   insn->perPatch = this->perPatch;
+
+   insn->postFactor = this->postFactor;
+
+   if (deep) {
+      if (!bb)
+         return;
+      Function *fn = bb->getFunction();
+      for (int d = 0; this->defExists(d); ++d)
+         insn->setDef(d, this->getDef(d)->clone(fn));
+   } else {
+      for (int d = 0; this->defExists(d); ++d)
+         insn->setDef(d, this->getDef(d));
+   }
+
+   for (int s = 0; this->srcExists(s); ++s)
+      insn->src[s].set(this->src[s]);
+
+   insn->predSrc = this->predSrc;
+   insn->flagsDef = this->flagsDef;
+   insn->flagsSrc = this->flagsSrc;
+}
+
+unsigned int
+Instruction::defCount(unsigned int mask) const
+{
+   unsigned int i, n;
+
+   for (n = 0, i = 0; this->defExists(i); ++i, mask >>= 1)
+      n += mask & 1;
+   return n;
+}
+
+unsigned int
+Instruction::srcCount(unsigned int mask) const
+{
+   unsigned int i, n;
+
+   for (n = 0, i = 0; this->srcExists(i); ++i, mask >>= 1)
+      n += mask & 1;
+   return n;
+}
+
+bool
+Instruction::setIndirect(int s, int dim, Value *value)
+{
+   int p = src[s].indirect[dim];
+
+   assert(this->srcExists(s));
+   if (p < 0) {
+      if (!value)
+         return true;
+      for (p = s + 1; this->srcExists(p); ++p);
+   }
+   assert(p < NV50_IR_MAX_SRCS);
+
+   src[p] = value;
+   src[p].usedAsPtr = (value != 0);
+   src[s].indirect[dim] = value ? p : -1;
+   return true;
+}
+
+bool
+Instruction::setPredicate(CondCode ccode, Value *value)
+{
+   cc = ccode;
+
+   if (!value) {
+      if (predSrc >= 0) {
+         src[predSrc] = 0;
+         predSrc = -1;
+      }
+      return true;
+   }
+
+   if (predSrc < 0) {
+      int s;
+      for (s = 0; this->srcExists(s); ++s)
+      assert(s < NV50_IR_MAX_SRCS);
+      predSrc = s;
+   }
+   src[predSrc] = value;
+   return true;
+}
+
+bool
+Instruction::writesPredicate() const
+{
+   for (int d = 0; d < 2 && def[d].exists(); ++d)
+      if (def[d].exists() &&
+          (getDef(d)->inFile(FILE_PREDICATE) || getDef(d)->inFile(FILE_FLAGS)))
+         return true;
+   return false;
+}
+
+static bool
+insnCheckCommutation(const Instruction *a, const Instruction *b)
+{
+   for (int d = 0; a->defExists(d); ++d)
+      for (int s = 0; b->srcExists(s); ++s)
+         if (a->getDef(d)->interfers(b->getSrc(s)))
+            return false;
+   return true;
+}
+
+bool
+Instruction::isCommutationLegal(const Instruction *i) const
+{
+   bool ret = true;
+   ret = ret && insnCheckCommutation(this, i);
+   ret = ret && insnCheckCommutation(i, this);
+   return ret;
+}
+
+TexInstruction::TexInstruction(Function *fn, operation op)
+   : Instruction(fn, op, TYPE_F32)
+{
+   memset(&tex, 0, sizeof(tex));
+
+   tex.rIndirectSrc = -1;
+   tex.sIndirectSrc = -1;
+}
+
+TexInstruction::~TexInstruction()
+{
+   for (int c = 0; c < 3; ++c) {
+      dPdx[c].set(NULL);
+      dPdy[c].set(NULL);
+   }
+}
+
+Instruction *
+TexInstruction::clone(bool deep) const
+{
+   TexInstruction *tex = new_TexInstruction(bb->getFunction(), op);
+   cloneBase(tex, deep);
+
+   tex->tex = this->tex;
+
+   if (op == OP_TXD) {
+      for (unsigned int c = 0; c < tex->tex.target.getDim(); ++c) {
+         tex->dPdx[c].set(dPdx[c]);
+         tex->dPdy[c].set(dPdy[c]);
+      }
+   }
+
+   return tex;
+}
+
+const struct TexInstruction::Target::Desc TexInstruction::Target::descTable[] =
+{
+   { "1D",                1, 1, false, false, false },
+   { "2D",                2, 2, false, false, false },
+   { "2D_MS",             2, 2, false, false, false },
+   { "3D",                3, 3, false, false, false },
+   { "CUBE",              2, 3, false, true,  false },
+   { "1D_SHADOW",         1, 1, false, false, true  },
+   { "2D_SHADOW",         2, 2, false, false, true  },
+   { "CUBE_SHADOW",       2, 3, false, true,  true  },
+   { "1D_ARRAY",          1, 2, true,  false, false },
+   { "2D_ARRAY",          2, 3, true,  false, false },
+   { "2D_MS_ARRAY",       2, 3, true,  false, false },
+   { "CUBE_ARRAY",        2, 3, true,  true,  false },
+   { "1D_ARRAY_SHADOW",   1, 2, true,  false, true  },
+   { "2D_ARRAY_SHADOW",   2, 3, true,  false, true  },
+   { "RECT",              2, 2, false, false, false },
+   { "RECT_SHADOW",       2, 2, false, false, true  },
+   { "CUBE_ARRAY_SHADOW", 2, 4, true,  true,  true  },
+   { "BUFFER",            1, 1, false, false, false },
+};
+
+CmpInstruction::CmpInstruction(Function *fn, operation op)
+   : Instruction(fn, op, TYPE_F32)
+{
+   setCond = CC_ALWAYS;
+}
+
+Instruction *
+CmpInstruction::clone(bool deep) const
+{
+   CmpInstruction *cmp = new_CmpInstruction(bb->getFunction(), op);
+   cloneBase(cmp, deep);
+   cmp->setCond = setCond;
+   cmp->dType = dType;
+   return cmp;
+}
+
+FlowInstruction::FlowInstruction(Function *fn, operation op,
+                                 BasicBlock *targ)
+   : Instruction(fn, op, TYPE_NONE)
+{
+   target.bb = targ;
+
+   if (op == OP_BRA ||
+       op == OP_CONT || op == OP_BREAK ||
+       op == OP_RET || op == OP_EXIT)
+      terminator = 1;
+   else
+   if (op == OP_JOIN)
+      terminator = targ ? 1 : 0;
+
+   allWarp = absolute = limit = 0;
+}
+
+Program::Program(Type type, Target *arch)
+   : progType(type),
+     target(arch),
+     mem_Instruction(sizeof(Instruction), 6),
+     mem_CmpInstruction(sizeof(CmpInstruction), 4),
+     mem_TexInstruction(sizeof(TexInstruction), 4),
+     mem_FlowInstruction(sizeof(FlowInstruction), 4),
+     mem_LValue(sizeof(LValue), 8),
+     mem_Symbol(sizeof(Symbol), 7),
+     mem_ImmediateValue(sizeof(ImmediateValue), 7)
+{
+   code = NULL;
+   binSize = 0;
+
+   maxGPR = -1;
+
+   main = new Function(this, "MAIN");
+
+   dbgFlags = 0;
+}
+
+Program::~Program()
+{
+   if (main)
+      delete main;
+}
+
+void Program::releaseInstruction(Instruction *insn)
+{
+   // TODO: make this not suck so much
+
+   insn->~Instruction();
+
+   if (insn->asCmp())
+      mem_CmpInstruction.release(insn);
+   else
+   if (insn->asTex())
+      mem_TexInstruction.release(insn);
+   else
+   if (insn->asFlow())
+      mem_FlowInstruction.release(insn);
+   else
+      mem_Instruction.release(insn);
+}
+
+void Program::releaseValue(Value *value)
+{
+   if (value->asLValue())
+      mem_LValue.release(value);
+   else
+   if (value->asImm())
+      mem_ImmediateValue.release(value);
+   else
+   if (value->asSym())
+      mem_Symbol.release(value);
+}
+
+
+} // namespace nv50_ir
+
+extern "C" {
+
+static void
+nv50_ir_init_prog_info(struct nv50_ir_prog_info *info)
+{
+   info->io.clipDistance = 0xff;
+   info->io.pointSize = 0xff;
+   info->io.edgeFlagIn = 0xff;
+   info->io.edgeFlagOut = 0xff;
+   info->io.fragDepth = 0xff;
+   info->io.sampleMask = 0xff;
+   info->io.backFaceColor[0] = info->io.backFaceColor[1] = 0xff;
+}
+   
+int
+nv50_ir_generate_code(struct nv50_ir_prog_info *info)
+{
+   int ret = 0;
+
+   nv50_ir::Program::Type type;
+
+   nv50_ir_init_prog_info(info);
+
+#define PROG_TYPE_CASE(a, b)                                      \
+   case PIPE_SHADER_##a: type = nv50_ir::Program::TYPE_##b; break
+
+   switch (info->type) {
+   PROG_TYPE_CASE(VERTEX, VERTEX);
+// PROG_TYPE_CASE(HULL, TESSELLATION_CONTROL);
+// PROG_TYPE_CASE(DOMAIN, TESSELLATION_EVAL);
+   PROG_TYPE_CASE(GEOMETRY, GEOMETRY);
+   PROG_TYPE_CASE(FRAGMENT, FRAGMENT);
+   default:
+      type = nv50_ir::Program::TYPE_COMPUTE;
+      break;
+   }
+   INFO_DBG(info->dbgFlags, VERBOSE, "translating program of type %u\n", type);
+
+   nv50_ir::Target *targ = nv50_ir::Target::create(info->target);
+   if (!targ)
+      return -1;
+
+   nv50_ir::Program *prog = new nv50_ir::Program(type, targ);
+   if (!prog)
+      return -1;
+   prog->dbgFlags = info->dbgFlags;
+
+   switch (info->bin.sourceRep) {
+#if 0
+   case PIPE_IR_LLVM:
+   case PIPE_IR_GLSL:
+      return -1;
+   case PIPE_IR_SM4:
+      ret = prog->makeFromSM4(info) ? 0 : -2;
+      break;
+   case PIPE_IR_TGSI:
+#endif
+   default:
+      ret = prog->makeFromTGSI(info) ? 0 : -2;
+      break;
+   }
+   if (ret < 0)
+      goto out;
+   if (prog->dbgFlags & NV50_IR_DEBUG_VERBOSE)
+      prog->print();
+
+   prog->getTarget()->runLegalizePass(prog, nv50_ir::CG_STAGE_PRE_SSA);
+
+   prog->convertToSSA();
+
+   if (prog->dbgFlags & NV50_IR_DEBUG_VERBOSE)
+      prog->print();
+
+   prog->optimizeSSA(info->optLevel);
+   prog->getTarget()->runLegalizePass(prog, nv50_ir::CG_STAGE_SSA);
+
+   if (prog->dbgFlags & NV50_IR_DEBUG_BASIC)
+      prog->print();
+
+   if (!prog->registerAllocation()) {
+      ret = -4;
+      goto out;
+   }
+   prog->getTarget()->runLegalizePass(prog, nv50_ir::CG_STAGE_POST_RA);
+
+   prog->optimizePostRA(info->optLevel);
+
+   if (!prog->emitBinary(info)) {
+      ret = -5;
+      goto out;
+   }
+
+out:
+   INFO_DBG(prog->dbgFlags, VERBOSE, "nv50_ir_generate_code: ret = %i\n", ret);
+
+   info->bin.maxGPR = prog->maxGPR;
+   info->bin.code = prog->code;
+   info->bin.codeSize = prog->binSize;
+
+   delete prog;
+   nv50_ir::Target::destroy(targ);
+
+   return ret;
+}
+
+} // extern "C"
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir.h b/src/gallium/drivers/nv50/codegen/nv50_ir.h
new file mode 100644
index 00000000000..6eef1abb69d
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir.h
@@ -0,0 +1,1049 @@
+
+#ifndef __NV50_IR_H__
+#define __NV50_IR_H__
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+#include "nv50_ir_util.h"
+#include "nv50_ir_graph.h"
+
+#include "nv50_ir_driver.h"
+
+namespace nv50_ir {
+
+enum operation
+{
+   OP_NOP = 0,
+   OP_PHI,
+   OP_UNION, // unify a new definition and several source values
+   OP_SPLIT, // $r0d -> { $r0, $r1 } ($r0d and $r0/$r1 will be coalesced)
+   OP_MERGE, // opposite of split, e.g. combine 2 32 bit into a 64 bit value
+   OP_CONSTRAINT, // copy values into consecutive registers
+   OP_MOV,
+   OP_LOAD,
+   OP_STORE,
+   OP_ADD,
+   OP_SUB,
+   OP_MUL,
+   OP_DIV,
+   OP_MOD,
+   OP_MAD,
+   OP_FMA,
+   OP_SAD, // abs(src0 - src1) + src2
+   OP_ABS,
+   OP_NEG,
+   OP_NOT,
+   OP_AND,
+   OP_OR,
+   OP_XOR,
+   OP_SHL,
+   OP_SHR,
+   OP_MAX,
+   OP_MIN,
+   OP_SAT,  // CLAMP(f32, 0.0, 1.0)
+   OP_CEIL,
+   OP_FLOOR,
+   OP_TRUNC,
+   OP_CVT,
+   OP_SET_AND, // dst = (src0 CMP src1) & src2
+   OP_SET_OR,
+   OP_SET_XOR,
+   OP_SET,
+   OP_SELP, // dst = src2 ? src0 : src1
+   OP_SLCT, // dst = (src2 CMP 0) ? src0 : src1
+   OP_RCP,
+   OP_RSQ,
+   OP_LG2,
+   OP_SIN,
+   OP_COS,
+   OP_EX2,
+   OP_EXP, // exponential (base M_E)
+   OP_LOG, // natural logarithm
+   OP_PRESIN,
+   OP_PREEX2,
+   OP_SQRT,
+   OP_POW,
+   OP_BRA,
+   OP_CALL,
+   OP_RET,
+   OP_CONT,
+   OP_BREAK,
+   OP_PRERET,
+   OP_PRECONT,
+   OP_PREBREAK,
+   OP_BRKPT,     // breakpoint (not related to loops)
+   OP_JOINAT,    // push control flow convergence point
+   OP_JOIN,      // converge
+   OP_DISCARD,
+   OP_EXIT,
+   OP_MEMBAR,
+   OP_VFETCH, // indirection 0 in attribute space, indirection 1 is vertex base
+   OP_PFETCH, // fetch base address of vertex src0 (immediate) [+ src1]
+   OP_EXPORT,
+   OP_LINTERP,
+   OP_PINTERP,
+   OP_EMIT,    // emit vertex
+   OP_RESTART, // restart primitive
+   OP_TEX,
+   OP_TXB, // texture bias
+   OP_TXL, // texure lod
+   OP_TXF, // texel fetch
+   OP_TXQ, // texture size query
+   OP_TXD, // texture derivatives
+   OP_TXG, // texture gather
+   OP_TEXCSAA,
+   OP_SULD, // surface load
+   OP_SUST, // surface store
+   OP_DFDX,
+   OP_DFDY,
+   OP_RDSV, // read system value
+   OP_WRSV, // write system value
+   OP_PIXLD,
+   OP_QUADOP,
+   OP_QUADON,
+   OP_QUADPOP,
+   OP_POPCNT, // bitcount(src0 & src1)
+   OP_INSBF,  // insert first src1[8:15] bits of src0 into src2 at src1[0:7]
+   OP_EXTBF,
+   OP_LAST
+};
+
+#define NV50_IR_SUBOP_MUL_HIGH     1
+#define NV50_IR_SUBOP_EMIT_RESTART 1
+#define NV50_IR_SUBOP_LDC_IL       1
+#define NV50_IR_SUBOP_LDC_IS       2
+#define NV50_IR_SUBOP_LDC_ISL      3
+
+enum DataType
+{
+   TYPE_NONE,
+   TYPE_U8,
+   TYPE_S8,
+   TYPE_U16,
+   TYPE_S16,
+   TYPE_U32,
+   TYPE_S32,
+   TYPE_U64, // 64 bit operations are only lowered after register allocation
+   TYPE_S64,
+   TYPE_F16,
+   TYPE_F32,
+   TYPE_F64,
+   TYPE_B96,
+   TYPE_B128
+};
+
+enum CondCode
+{
+   CC_FL = 0,
+   CC_NEVER = CC_FL, // when used with FILE_FLAGS
+   CC_LT = 1,
+   CC_EQ = 2,
+   CC_NOT_P = CC_EQ, // when used with FILE_PREDICATE
+   CC_LE = 3,
+   CC_GT = 4,
+   CC_NE = 5,
+   CC_P  = CC_NE,
+   CC_GE = 6,
+   CC_TR = 7,
+   CC_ALWAYS = CC_TR,
+   CC_U  = 8,
+   CC_LTU = 9,
+   CC_EQU = 10,
+   CC_LEU = 11,
+   CC_GTU = 12,
+   CC_NEU = 13,
+   CC_GEU = 14,
+   CC_NO = 0x10,
+   CC_NC = 0x11,
+   CC_NS = 0x12,
+   CC_NA = 0x13,
+   CC_A  = 0x14,
+   CC_S  = 0x15,
+   CC_C  = 0x16,
+   CC_O  = 0x17
+};
+
+enum RoundMode
+{
+   ROUND_N, // nearest
+   ROUND_M, // towards -inf
+   ROUND_Z, // towards 0
+   ROUND_P, // towards +inf
+   ROUND_NI, // nearest integer
+   ROUND_MI, // to integer towards -inf
+   ROUND_ZI, // to integer towards 0
+   ROUND_PI, // to integer towards +inf
+};
+
+enum CacheMode
+{
+   CACHE_CA,            // cache at all levels
+   CACHE_WB = CACHE_CA, // cache write back
+   CACHE_CG,            // cache at global level
+   CACHE_CS,            // cache streaming
+   CACHE_CV,            // cache as volatile
+   CACHE_WT = CACHE_CV  // cache write-through
+};
+
+enum DataFile
+{
+   FILE_NULL = 0,
+   FILE_GPR,
+   FILE_PREDICATE,       // boolean predicate
+   FILE_FLAGS,           // zero/sign/carry/overflow bits
+   FILE_ADDRESS,
+   FILE_IMMEDIATE,
+   FILE_MEMORY_CONST,
+   FILE_SHADER_INPUT,
+   FILE_SHADER_OUTPUT,
+   FILE_MEMORY_GLOBAL,
+   FILE_MEMORY_SHARED,
+   FILE_MEMORY_LOCAL,
+   FILE_SYSTEM_VALUE,
+   DATA_FILE_COUNT
+};
+
+enum TexTarget
+{
+   TEX_TARGET_1D,
+   TEX_TARGET_2D,
+   TEX_TARGET_2D_MS,
+   TEX_TARGET_3D,
+   TEX_TARGET_CUBE,
+   TEX_TARGET_1D_SHADOW,
+   TEX_TARGET_2D_SHADOW,
+   TEX_TARGET_CUBE_SHADOW,
+   TEX_TARGET_1D_ARRAY,
+   TEX_TARGET_2D_ARRAY,
+   TEX_TARGET_2D_MS_ARRAY,
+   TEX_TARGET_CUBE_ARRAY,
+   TEX_TARGET_1D_ARRAY_SHADOW,
+   TEX_TARGET_2D_ARRAY_SHADOW,
+   TEX_TARGET_RECT,
+   TEX_TARGET_RECT_SHADOW,
+   TEX_TARGET_CUBE_ARRAY_SHADOW,
+   TEX_TARGET_BUFFER,
+   TEX_TARGET_COUNT
+};
+
+enum SVSemantic
+{
+   SV_POSITION, // WPOS
+   SV_VERTEX_ID,
+   SV_INSTANCE_ID,
+   SV_INVOCATION_ID,
+   SV_PRIMITIVE_ID,
+   SV_VERTEX_COUNT, // gl_PatchVerticesIn
+   SV_LAYER,
+   SV_VIEWPORT_INDEX,
+   SV_YDIR,
+   SV_FACE,
+   SV_POINT_SIZE,
+   SV_POINT_COORD,
+   SV_CLIP_DISTANCE,
+   SV_SAMPLE_INDEX,
+   SV_TESS_FACTOR,
+   SV_TESS_COORD,
+   SV_TID,
+   SV_CTAID,
+   SV_NTID,
+   SV_GRIDID,
+   SV_NCTAID,
+   SV_LANEID,
+   SV_PHYSID,
+   SV_NPHYSID,
+   SV_CLOCK,
+   SV_LBASE,
+   SV_SBASE,
+   SV_UNDEFINED,
+   SV_LAST
+};
+
+class Program;
+class Function;
+class BasicBlock;
+
+class Target;
+
+class Instruction;
+class CmpInstruction;
+class TexInstruction;
+class FlowInstruction;
+
+class Value;
+class LValue;
+class Symbol;
+class ImmediateValue;
+
+struct Storage
+{
+   DataFile file;
+   int8_t fileIndex; // signed, may be indirect for CONST[]
+   uint8_t size; // this should match the Instruction type's size
+   DataType type; // mainly for pretty printing
+   union {
+      uint64_t u64;    // immediate values
+      uint32_t u32;
+      uint16_t u16;
+      uint8_t u8;
+      int64_t s64;
+      int32_t s32;
+      int16_t s16;
+      int8_t s8;
+      float f32;
+      double f64;
+      int32_t offset; // offset from 0 (base of address space)
+      int32_t id;     // register id (< 0 if virtual/unassigned)
+      struct {
+         SVSemantic sv;
+         int index;
+      } sv;
+   } data;
+};
+
+// precedence: NOT after SAT after NEG after ABS
+#define NV50_IR_MOD_ABS (1 << 0)
+#define NV50_IR_MOD_NEG (1 << 1)
+#define NV50_IR_MOD_SAT (1 << 2)
+#define NV50_IR_MOD_NOT (1 << 3)
+#define NV50_IR_MOD_NEG_ABS (NV50_IR_MOD_NEG | NV50_IR_MOD_ABS)
+
+#define NV50_IR_INTERP_MODE_MASK   0x3
+#define NV50_IR_INTERP_LINEAR      (0 << 0)
+#define NV50_IR_INTERP_PERSPECTIVE (1 << 0)
+#define NV50_IR_INTERP_FLAT        (2 << 0)
+#define NV50_IR_INTERP_SC          (3 << 0) // what exactly is that ?
+#define NV50_IR_INTERP_SAMPLE_MASK 0xc
+#define NV50_IR_INTERP_DEFAULT     (0 << 2)
+#define NV50_IR_INTERP_CENTROID    (1 << 2)
+#define NV50_IR_INTERP_OFFSET      (2 << 2)
+#define NV50_IR_INTERP_SAMPLEID    (3 << 2)
+
+// do we really want this to be a class ?
+class Modifier
+{
+public:
+   Modifier() : bits(0) { }
+   Modifier(unsigned int m) : bits(m) { }
+   Modifier(operation op);
+
+   // @return new Modifier applying a after b (asserts if unrepresentable)
+   Modifier operator*(const Modifier) const;
+   Modifier operator==(const Modifier m) const { return m.bits == bits; }
+   Modifier operator!=(const Modifier m) const { return m.bits != bits; }
+
+   inline Modifier operator&(const Modifier m) const { return bits & m.bits; }
+   inline Modifier operator|(const Modifier m) const { return bits | m.bits; }
+   inline Modifier operator^(const Modifier m) const { return bits ^ m.bits; }
+
+   operation getOp() const;
+
+   inline int neg() const { return (bits & NV50_IR_MOD_NEG) ? 1 : 0; }
+   inline int abs() const { return (bits & NV50_IR_MOD_ABS) ? 1 : 0; }
+
+   inline operator bool() { return bits ? true : false; }
+
+   void applyTo(ImmediateValue &imm) const;
+
+   int print(char *buf, size_t size) const;
+
+private:
+   uint8_t bits;
+};
+
+class ValueRef
+{
+public:
+   ValueRef();
+   ~ValueRef();
+
+   inline ValueRef& operator=(Value *val) { this->set(val); return *this; }
+
+   inline bool exists() const { return value != NULL; }
+
+   void set(Value *);
+   void set(const ValueRef&);
+   inline Value *get() const { return value; }
+   inline Value *rep() const;
+
+   inline Instruction *getInsn() const { return insn; }
+   inline void setInsn(Instruction *inst) { insn = inst; }
+
+   inline bool isIndirect(int dim) const { return indirect[dim] >= 0; }
+   inline const ValueRef *getIndirect(int dim) const;
+
+   inline DataFile getFile() const;
+   inline unsigned getSize() const;
+
+   // SSA: return eventual (traverse MOVs) literal value, if it exists
+   ImmediateValue *getImmediate() const;
+
+   class Iterator
+   {
+   public:
+      Iterator(ValueRef *ref) : pos(ref), ini(ref) { }
+
+      inline ValueRef *get() const { return pos; }
+      inline bool end() const { return pos == NULL; }
+      inline void next() { pos = (pos->next != ini) ? pos->next : 0; }
+
+   private:
+      ValueRef *pos, *ini;
+   };
+
+   inline Iterator iterator() { return Iterator(this); }
+
+public:
+   Modifier mod;
+   int8_t indirect[2]; // >= 0 if relative to lvalue in insn->src[indirect[i]]
+   uint8_t swizzle;
+
+   bool usedAsPtr; // for printing
+
+private:
+   Value *value;
+   Instruction *insn;
+   ValueRef *next; // to link uses of the value
+   ValueRef *prev;
+};
+
+class ValueDef
+{
+public:
+   ValueDef();
+   ~ValueDef();
+
+   inline ValueDef& operator=(Value *val) { this->set(val); return *this; }
+
+   inline bool exists() const { return value != NULL; }
+
+   inline Value *get() const { return value; }
+   inline Value *rep() const;
+   void set(Value *);
+   void replace(Value *, bool doSet); // replace all uses of the old value
+
+   inline Instruction *getInsn() const { return insn; }
+   inline void setInsn(Instruction *inst) { insn = inst; }
+
+   inline DataFile getFile() const;
+   inline unsigned getSize() const;
+
+   // HACK: save the pre-SSA value in 'prev', in SSA we don't need the def list
+   //  but we'll use it again for coalescing in register allocation
+   inline void setSSA(LValue *);
+   inline const LValue *preSSA() const;
+   inline void restoreDefList(); // after having been abused for SSA hack
+   void mergeDefs(ValueDef *);
+
+   class Iterator
+   {
+   public:
+      Iterator(ValueDef *def) : pos(def), ini(def) { }
+
+      inline ValueDef *get() const { return pos; }
+      inline bool end() const { return pos == NULL; }
+      inline void next() { pos = (pos->next != ini) ? pos->next : NULL; }
+
+   private:
+      ValueDef *pos, *ini;
+   };
+
+   inline Iterator iterator() { return Iterator(this); }
+
+private:
+   Value *value;   // should make this LValue * ...
+   Instruction *insn;
+   ValueDef *next; // circular list of all definitions of the same value
+   ValueDef *prev;
+};
+
+class Value
+{
+public:
+   Value();
+
+   virtual Value *clone(Function *) const { return NULL; }
+
+   virtual int print(char *, size_t, DataType ty = TYPE_NONE) const = 0;
+
+   virtual bool equals(const Value *, bool strict = false) const;
+   virtual bool interfers(const Value *) const;
+
+   inline Instruction *getUniqueInsn() const;
+   inline Instruction *getInsn() const; // use when uniqueness is certain
+
+   inline int refCount() { return refCnt; }
+   inline int ref() { return ++refCnt; }
+   inline int unref() { --refCnt; assert(refCnt >= 0); return refCnt; }
+
+   inline LValue *asLValue();
+   inline Symbol *asSym();
+   inline ImmediateValue *asImm();
+   inline const Symbol *asSym() const;
+   inline const ImmediateValue *asImm() const;
+
+   bool coalesce(Value *, bool force = false);
+
+   inline bool inFile(DataFile f) { return reg.file == f; }
+
+   static inline Value *get(Iterator&);
+
+protected:
+   int refCnt;
+
+   friend class ValueDef;
+   friend class ValueRef;
+
+public:
+   int id;
+   ValueRef *uses;
+   ValueDef *defs;
+   Storage reg;
+
+   // TODO: these should be in LValue:
+   Interval livei;
+   Value *join;
+};
+
+class LValue : public Value
+{
+public:
+   LValue(Function *, DataFile file);
+   LValue(Function *, LValue *);
+
+   virtual Value *clone(Function *) const;
+
+   virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
+
+public:
+   unsigned ssa : 1;
+
+   int affinity;
+};
+
+class Symbol : public Value
+{
+public:
+   Symbol(Program *, DataFile file = FILE_MEMORY_CONST, ubyte fileIdx = 0);
+
+   virtual Value *clone(Function *) const;
+
+   virtual bool equals(const Value *that, bool strict) const;
+
+   virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
+
+   // print with indirect values
+   int print(char *, size_t, Value *, Value *, DataType ty = TYPE_NONE) const;
+
+   inline void setFile(DataFile file, ubyte fileIndex = 0)
+   {
+      reg.file = file;
+      reg.fileIndex = fileIndex;
+   }
+
+   inline void setOffset(int32_t offset);
+   inline void setAddress(Symbol *base, int32_t offset);
+   inline void setSV(SVSemantic sv, uint32_t idx = 0);
+
+   inline const Symbol *getBase() const { return baseSym; }
+
+private:
+   Symbol *baseSym; // array base for Symbols representing array elements
+};
+
+class ImmediateValue : public Value
+{
+public:
+   ImmediateValue(Program *, uint32_t);
+   ImmediateValue(Program *, float);
+   ImmediateValue(Program *, double);
+
+   // NOTE: not added to program with
+   ImmediateValue(const ImmediateValue *, DataType ty);
+
+   virtual bool equals(const Value *that, bool strict) const;
+
+   // these only work if 'type' is valid (we mostly use untyped literals):
+   bool isInteger(const int ival) const; // ival is cast to this' type
+   bool isNegative() const;
+   bool isPow2() const;
+
+   void applyLog2();
+
+   // for constant folding:
+   ImmediateValue operator+(const ImmediateValue&) const;
+   ImmediateValue operator-(const ImmediateValue&) const;
+   ImmediateValue operator*(const ImmediateValue&) const;
+   ImmediateValue operator/(const ImmediateValue&) const;
+
+   bool compare(CondCode cc, float fval) const;
+
+   virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
+};
+
+
+#define NV50_IR_MAX_DEFS 4
+#define NV50_IR_MAX_SRCS 8
+
+class Instruction
+{
+public:
+   Instruction();
+   Instruction(Function *, operation, DataType);
+   virtual ~Instruction();
+
+   virtual Instruction *clone(bool deep) const;
+
+   inline void setDef(int i, Value *val) { def[i].set(val); }
+   inline void setSrc(int s, Value *val) { src[s].set(val); }
+   void setSrc(int s, ValueRef&);
+   void swapSources(int a, int b);
+   bool setIndirect(int s, int dim, Value *);
+
+   inline Value *getDef(int d) const { return def[d].get(); }
+   inline Value *getSrc(int s) const { return src[s].get(); }
+   inline Value *getIndirect(int s, int dim) const;
+
+   inline bool defExists(int d) const { return d < 4 && def[d].exists(); }
+   inline bool srcExists(int s) const { return s < 8 && src[s].exists(); }
+
+   inline bool constrainedDefs() const { return def[1].exists(); }
+
+   bool setPredicate(CondCode ccode, Value *);
+   inline Value *getPredicate() const;
+   bool writesPredicate() const;
+
+   unsigned int defCount(unsigned int mask) const;
+   unsigned int srcCount(unsigned int mask) const;
+
+   // save & remove / set indirect[0,1] and predicate source
+   void takeExtraSources(int s, Value *[3]);
+   void putExtraSources(int s, Value *[3]);
+
+   inline void setType(DataType type) { dType = sType = type; }
+
+   inline void setType(DataType dtype, DataType stype)
+   {
+      dType = dtype;
+      sType = stype;
+   }
+
+   inline bool isPseudo() const { return op < OP_MOV; }
+   bool isDead() const;
+   bool isNop() const;
+   bool isCommutationLegal(const Instruction *) const; // must be adjacent !
+   bool isActionEqual(const Instruction *) const;
+   bool isResultEqual(const Instruction *) const;
+
+   void print() const;
+
+   inline CmpInstruction *asCmp();
+   inline TexInstruction *asTex();
+   inline FlowInstruction *asFlow();
+   inline const TexInstruction *asTex() const;
+   inline const CmpInstruction *asCmp() const;
+   inline const FlowInstruction *asFlow() const;
+
+public:
+   Instruction *next;
+   Instruction *prev;
+   int id;
+   int serial; // CFG order
+
+   operation op;
+   DataType dType; // destination or defining type
+   DataType sType; // source or secondary type
+   CondCode cc;
+   RoundMode rnd;
+   CacheMode cache;
+
+   uint8_t subOp; // quadop, 1 for mul-high, etc.
+
+   unsigned encSize    : 4; // encoding size in bytes
+   unsigned saturate   : 1; // to [0.0f, 1.0f]
+   unsigned join       : 1; // converge control flow (use OP_JOIN until end)
+   unsigned fixed      : 1; // prevent dead code elimination
+   unsigned terminator : 1; // end of basic block
+   unsigned atomic     : 1;
+   unsigned ftz        : 1; // flush denormal to zero
+   unsigned dnz        : 1; // denormals, NaN are zero
+   unsigned ipa        : 4; // interpolation mode
+   unsigned lanes      : 4;
+   unsigned perPatch   : 1;
+   unsigned exit       : 1; // terminate program after insn
+
+   int8_t postFactor; // MUL/DIV(if < 0) by 1 << postFactor
+
+   int8_t predSrc;
+   int8_t flagsDef;
+   int8_t flagsSrc;
+
+   // NOTE: should make these pointers, saves space and work on shuffling
+   ValueDef def[NV50_IR_MAX_DEFS]; // no gaps !
+   ValueRef src[NV50_IR_MAX_SRCS]; // no gaps !
+
+   BasicBlock *bb;
+
+   // instruction specific methods:
+   // (don't want to subclass, would need more constructors and memory pools)
+public:
+   inline void setInterpolate(unsigned int mode) { ipa = mode; }
+
+   unsigned int getInterpMode() const { return ipa & 0x3; }
+   unsigned int getSampleMode() const { return ipa & 0xc; }
+
+private:
+   void init();
+protected:
+   void cloneBase(Instruction *clone, bool deep) const;
+};
+
+enum TexQuery
+{
+   TXQ_DIMS,
+   TXQ_TYPE,
+   TXQ_SAMPLE_POSITION,
+   TXQ_FILTER,
+   TXQ_LOD,
+   TXQ_WRAP,
+   TXQ_BORDER_COLOUR
+};
+
+class TexInstruction : public Instruction
+{
+public:
+   class Target
+   {
+   public:
+      Target(TexTarget targ = TEX_TARGET_2D) : target(targ) { }
+
+      const char *getName() const { return descTable[target].name; }
+      unsigned int getArgCount() const { return descTable[target].argc; }
+      unsigned int getDim() const { return descTable[target].dim; }
+      int isArray() const { return descTable[target].array ? 1 : 0; }
+      int isCube() const { return descTable[target].cube ? 1 : 0; }
+      int isShadow() const { return descTable[target].shadow ? 1 : 0; }
+
+      Target& operator=(TexTarget targ)
+      {
+         assert(targ < TEX_TARGET_COUNT);
+         return *this;
+      }
+
+      inline bool operator==(TexTarget targ) const { return target == targ; }
+
+   private:
+      struct Desc
+      {
+         char name[19];
+         uint8_t dim;
+         uint8_t argc;
+         bool array;
+         bool cube;
+         bool shadow;
+      };
+
+      static const struct Desc descTable[TEX_TARGET_COUNT];
+
+   private:
+      enum TexTarget target;
+   };
+
+public:
+   TexInstruction(Function *, operation);
+   virtual ~TexInstruction();
+
+   virtual Instruction *clone(bool deep) const;
+
+   inline void setTexture(Target targ, uint8_t r, uint8_t s)
+   {
+      tex.r = r;
+      tex.s = s;
+      tex.target = targ;
+   }
+
+   inline Value *getIndirectR() const;
+   inline Value *getIndirectS() const;
+
+public:
+   struct {
+      Target target;
+
+      uint8_t r;
+      int8_t rIndirectSrc;
+      uint8_t s;
+      int8_t sIndirectSrc;
+
+      uint8_t mask;
+      uint8_t gatherComp;
+
+      bool liveOnly; // only execute on live pixels of a quad (optimization)
+      bool levelZero;
+
+      int8_t useOffsets; // 0, 1, or 4 for textureGatherOffsets
+      int8_t offset[4][3];
+
+      enum TexQuery query;
+   } tex;
+
+   ValueRef dPdx[3];
+   ValueRef dPdy[3];
+};
+
+class CmpInstruction : public Instruction
+{
+public:
+   CmpInstruction(Function *, operation);
+
+   virtual Instruction *clone(bool deep) const;
+
+   void setCondition(CondCode cond) { setCond = cond; }
+   CondCode getCondition() const { return setCond; }
+
+public:
+   CondCode setCond;
+};
+
+class FlowInstruction : public Instruction
+{
+public:
+   FlowInstruction(Function *, operation, BasicBlock *target);
+
+public:
+   unsigned allWarp  : 1;
+   unsigned absolute : 1;
+   unsigned limit    : 1;
+   unsigned builtin  : 1; // true for calls to emulation code
+
+   union {
+      BasicBlock *bb;
+      int builtin;
+      Function *fn;
+   } target;
+};
+
+class BasicBlock
+{
+public:
+   BasicBlock(Function *);
+   ~BasicBlock();
+
+   inline int getId() const { return id; }
+   inline unsigned int getInsnCount() const { return numInsns; }
+   inline bool isTerminated() const { return exit && exit->terminator; }
+
+   bool dominatedBy(BasicBlock *bb);
+   inline bool reachableBy(BasicBlock *by, BasicBlock *term);
+
+   // returns mask of conditional out blocks
+   // e.g. 3 for IF { .. } ELSE { .. } ENDIF, 1 for IF { .. } ENDIF
+   unsigned int initiatesSimpleConditional() const;
+
+public:
+   Function *getFunction() const { return func; }
+   Program *getProgram() const { return program; }
+
+   Instruction *getEntry() const { return entry; } // first non-phi instruction
+   Instruction *getPhi() const { return phi; }
+   Instruction *getFirst() const { return phi ? phi : entry; }
+   Instruction *getExit() const { return exit; }
+
+   void insertHead(Instruction *);
+   void insertTail(Instruction *);
+   void insertBefore(Instruction *, Instruction *);
+   void insertAfter(Instruction *, Instruction *);
+   void remove(Instruction *);
+   void permuteAdjacent(Instruction *, Instruction *);
+
+   BasicBlock *idom() const;
+
+   DLList& getDF() { return df; }
+   DLList::Iterator iterDF() { return df.iterator(); }
+
+   static inline BasicBlock *get(Iterator&);
+   static inline BasicBlock *get(Graph::Node *);
+
+public:
+   Graph::Node cfg; // first edge is branch *taken* (the ELSE branch)
+   Graph::Node dom;
+
+   BitSet liveSet;
+
+   uint32_t binPos;
+   uint32_t binSize;
+
+   Instruction *joinAt; // for quick reference
+
+   bool explicitCont; // loop headers: true if loop contains continue stmts
+
+private:
+   int id;
+   DLList df;
+
+   Instruction *phi;
+   Instruction *entry;
+   Instruction *exit;
+
+   unsigned int numInsns;
+
+private:
+   Function *func;
+   Program *program;
+};
+
+class Function
+{
+public:
+   Function(Program *, const char *name);
+   ~Function();
+
+   inline Program *getProgram() const { return prog; }
+   inline const char *getName() const { return name; }
+   inline int getId() const { return id; }
+
+   void print();
+   void printLiveIntervals() const;
+   void printCFGraph(const char *filePath);
+
+   bool setEntry(BasicBlock *);
+   bool setExit(BasicBlock *);
+
+   unsigned int orderInstructions(ArrayList&);
+
+   inline void add(BasicBlock *bb, int& id) { allBBlocks.insert(bb, id); }
+   inline void add(Instruction *insn, int& id) { allInsns.insert(insn, id); }
+   inline void add(LValue *lval, int& id) { allLValues.insert(lval, id); }
+
+   inline LValue *getLValue(int id);
+
+   bool convertToSSA();
+
+public:
+   Graph cfg;
+   Graph::Node *cfgExit;
+   Graph *domTree;
+   Graph::Node call; // node in the call graph
+
+   BasicBlock **bbArray; // BBs in emission order
+   int bbCount;
+
+   unsigned int loopNestingBound;
+   int regClobberMax;
+
+   uint32_t binPos;
+   uint32_t binSize;
+
+   ArrayList allBBlocks;
+   ArrayList allInsns;
+   ArrayList allLValues;
+
+private:
+   void buildLiveSetsPreSSA(BasicBlock *, const int sequence);
+
+private:
+   int id;
+   const char *const name;
+   Program *prog;
+};
+
+enum CGStage
+{
+   CG_STAGE_PRE_SSA,
+   CG_STAGE_SSA, // expected directly before register allocation
+   CG_STAGE_POST_RA
+};
+
+class Program
+{
+public:
+   enum Type
+   {
+      TYPE_VERTEX,
+      TYPE_TESSELLATION_CONTROL,
+      TYPE_TESSELLATION_EVAL,
+      TYPE_GEOMETRY,
+      TYPE_FRAGMENT,
+      TYPE_COMPUTE
+   };
+
+   Program(Type type, Target *targ);
+   ~Program();
+
+   void print();
+
+   Type getType() const { return progType; }
+
+   inline void add(Function *fn, int& id) { allFuncs.insert(fn, id); }
+   inline void add(Value *rval, int& id) { allRValues.insert(rval, id); }
+
+   bool makeFromTGSI(struct nv50_ir_prog_info *);
+   bool makeFromSM4(struct nv50_ir_prog_info *);
+   bool convertToSSA();
+   bool optimizeSSA(int level);
+   bool optimizePostRA(int level);
+   bool registerAllocation();
+   bool emitBinary(struct nv50_ir_prog_info *);
+
+   const Target *getTarget() const { return target; }
+
+private:
+   Type progType;
+   Target *target;
+
+public:
+   Function *main;
+   Graph calls;
+
+   ArrayList allFuncs;
+   ArrayList allRValues;
+
+   uint32_t *code;
+   uint32_t binSize;
+
+   int maxGPR;
+
+   MemoryPool mem_Instruction;
+   MemoryPool mem_CmpInstruction;
+   MemoryPool mem_TexInstruction;
+   MemoryPool mem_FlowInstruction;
+   MemoryPool mem_LValue;
+   MemoryPool mem_Symbol;
+   MemoryPool mem_ImmediateValue;
+
+   uint32_t dbgFlags;
+
+   void releaseInstruction(Instruction *);
+   void releaseValue(Value *);
+};
+
+// TODO: add const version
+class Pass
+{
+public:
+   bool run(Program *, bool ordered = false, bool skipPhi = false);
+   bool run(Function *, bool ordered = false, bool skipPhi = false);
+
+private:
+   // return false to continue with next entity on next higher level
+   virtual bool visit(Function *) { return true; }
+   virtual bool visit(BasicBlock *) { return true; }
+   virtual bool visit(Instruction *) { return false; }
+
+   bool doRun(Program *, bool ordered, bool skipPhi);
+   bool doRun(Function *, bool ordered, bool skipPhi);
+
+protected:
+   bool err;
+   Function *func;
+   Program *prog;
+};
+
+// =============================================================================
+
+#include "nv50_ir_inlines.h"
+
+} // namespace nv50_ir
+
+#endif // __NV50_IR_H__
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_bb.cpp b/src/gallium/drivers/nv50/codegen/nv50_ir_bb.cpp
new file mode 100644
index 00000000000..5bf08b37c51
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_bb.cpp
@@ -0,0 +1,409 @@
+
+#include "nv50_ir.h"
+
+namespace nv50_ir {
+
+Function::Function(Program *p, const char *fnName)
+   : call(this),
+     name(fnName),
+     prog(p)
+{
+   cfgExit = NULL;
+   domTree = NULL;
+
+   bbArray = NULL;
+   bbCount = 0;
+   loopNestingBound = 0;
+   regClobberMax = 0;
+
+   binPos = 0;
+   binSize = 0;
+
+   prog->add(this, id);
+}
+
+Function::~Function()
+{
+   if (domTree)
+      delete domTree;
+   if (bbArray)
+      delete[] bbArray;
+
+   for (ArrayList::Iterator BBs = allBBlocks.iterator(); !BBs.end(); BBs.next())
+      delete reinterpret_cast<BasicBlock *>(BBs.get());
+}
+
+BasicBlock::BasicBlock(Function *fn) : cfg(this), dom(this), func(fn)
+{
+   program = func->getProgram();
+
+   joinAt = phi = entry = exit = NULL;
+
+   numInsns = 0;
+   binPos = 0;
+   binSize = 0;
+
+   explicitCont = false;
+
+   func->add(this, this->id);
+}
+
+BasicBlock::~BasicBlock()
+{
+   // nothing yet
+}
+
+BasicBlock *
+BasicBlock::idom() const
+{
+   Graph::Node *dn = dom.parent();
+   return dn ? BasicBlock::get(dn) : NULL;
+}
+
+void
+BasicBlock::insertHead(Instruction *inst)
+{
+   assert(inst->next == 0 && inst->prev == 0);
+
+   if (inst->op == OP_PHI) {
+      if (phi) {
+         insertBefore(phi, inst);
+      } else {
+         if (entry) {
+            insertBefore(entry, phi);
+         } else {
+            assert(!exit);
+            phi = exit = inst;
+            inst->bb = this;
+            ++numInsns;
+         }
+      }
+   } else {
+      if (entry) {
+         insertBefore(entry, inst);
+      } else {
+         if (phi) {
+            insertAfter(phi, inst);
+         } else {
+            assert(!exit);
+            entry = exit = inst;
+            inst->bb = this;
+            ++numInsns;
+         }
+      }
+   }
+}
+
+void
+BasicBlock::insertTail(Instruction *inst)
+{
+   assert(inst->next == 0 && inst->prev == 0);
+
+   if (inst->op == OP_PHI) {
+      if (entry) {
+         insertBefore(entry, inst);
+      } else
+      if (exit) {
+         assert(phi);
+         insertAfter(exit, inst);
+      } else {
+         assert(!phi);
+         phi = exit = inst;
+         inst->bb = this;
+         ++numInsns;
+      }
+   } else {
+      if (exit) {
+         insertAfter(exit, inst);
+      } else {
+         assert(!phi);
+         entry = exit = inst;
+         inst->bb = this;
+         ++numInsns;
+      }
+   }
+}
+
+void
+BasicBlock::insertBefore(Instruction *q, Instruction *p)
+{
+   assert(p && q);
+
+   assert(p->next == 0 && p->prev == 0);
+
+   if (q == entry) {
+      if (p->op == OP_PHI) {
+         if (!phi)
+            phi = p;
+      } else {
+         entry = p;
+      }
+   } else
+   if (q == phi) {
+      assert(p->op == OP_PHI);
+      phi = p;
+   }
+
+   p->next = q;
+   p->prev = q->prev;
+   if (p->prev)
+      p->prev->next = p;
+   q->prev = p;
+
+   p->bb = this;
+   ++numInsns;
+}
+
+void
+BasicBlock::insertAfter(Instruction *p, Instruction *q)
+{
+   assert(p && q);
+   assert(q->op != OP_PHI || p->op == OP_PHI);
+
+   assert(q->next == 0 && q->prev == 0);
+
+   if (p == exit)
+      exit = q;
+   if (p->op == OP_PHI && q->op != OP_PHI)
+      entry = q;
+
+   q->prev = p;
+   q->next = p->next;
+   if (q->next)
+      q->next->prev = q;
+   p->next = q;
+
+   q->bb = this;
+   ++numInsns;
+}
+
+void
+BasicBlock::remove(Instruction *insn)
+{
+   assert(insn->bb == this);
+
+   if (insn->prev)
+      insn->prev->next = insn->next;
+
+   if (insn->next)
+      insn->next->prev = insn->prev;
+   else
+      exit = insn->prev;
+
+   if (insn == entry)
+      entry = insn->next ? insn->next : insn->prev;
+
+   if (insn == phi)
+      phi = (insn->next && insn->next->op == OP_PHI) ? insn->next : 0;
+
+   --numInsns;
+   insn->bb = NULL;
+   insn->next =
+   insn->prev = NULL;
+}
+
+void BasicBlock::permuteAdjacent(Instruction *a, Instruction *b)
+{
+   assert(a->bb == b->bb);
+
+   if (a->next != b) {
+      Instruction *i = a;
+      a = b;
+      b = i;
+   }
+   assert(a->next == b);
+   assert(a->op != OP_PHI && b->op != OP_PHI);
+
+   if (b == exit)
+      exit = a;
+   if (a == entry)
+      entry = b;
+
+   b->prev = a->prev;
+   a->next = b->next;
+   b->next = a;
+   a->prev = b;
+
+   if (b->prev)
+      b->prev->next = b;
+   if (a->prev)
+      a->next->prev = a;
+}
+
+bool
+BasicBlock::dominatedBy(BasicBlock *that)
+{
+   Graph::Node *bn = &that->dom;
+   Graph::Node *dn = &this->dom;
+
+   while (dn && dn != bn)
+      dn = dn->parent();
+
+   return dn != NULL;
+}
+
+unsigned int
+BasicBlock::initiatesSimpleConditional() const
+{
+   Graph::Node *out[2];
+   int n;
+   Graph::Edge::Type eR;
+
+   if (cfg.outgoingCount() != 2) // -> if and -> else/endif
+      return false;
+
+   n = 0;
+   for (Graph::EdgeIterator ei = cfg.outgoing(); !ei.end(); ei.next())
+      out[n++] = ei.getNode();
+   eR = out[1]->outgoing().getType();
+
+   // IF block is out edge to the right
+   if (eR == Graph::Edge::CROSS || eR == Graph::Edge::BACK)
+      return 0x2;
+
+   if (out[1]->outgoingCount() != 1) // 0 is IF { RET; }, >1 is more divergence
+      return 0x0;
+   // do they reconverge immediately ?
+   if (out[1]->outgoing().getNode() == out[0])
+      return 0x1;
+   if (out[0]->outgoingCount() == 1)
+      if (out[0]->outgoing().getNode() == out[1]->outgoing().getNode())
+         return 0x3;
+
+   return 0x0;
+}
+
+bool
+Function::setEntry(BasicBlock *bb)
+{
+   if (cfg.getRoot())
+      return false;
+   cfg.insert(&bb->cfg);
+   return true;
+}
+
+bool
+Function::setExit(BasicBlock *bb)
+{
+   if (cfgExit)
+      return false;
+   cfgExit = &bb->cfg;
+   return true;
+}
+
+unsigned int
+Function::orderInstructions(ArrayList &result)
+{
+   Iterator *iter;
+   for (iter = cfg.iteratorCFG(); !iter->end(); iter->next())
+      for (Instruction *insn = BasicBlock::get(*iter)->getFirst();
+           insn; insn = insn->next)
+         result.insert(insn, insn->serial);
+   cfg.putIterator(iter);
+   return result.getSize();
+}
+
+bool
+Pass::run(Program *prog, bool ordered, bool skipPhi)
+{
+   this->prog = prog;
+   err = false;
+   return doRun(prog, ordered, skipPhi);
+}
+
+bool
+Pass::doRun(Program *prog, bool ordered, bool skipPhi)
+{
+   for (ArrayList::Iterator fi = prog->allFuncs.iterator();
+        !fi.end(); fi.next()) {
+      Function *fn = reinterpret_cast<Function *>(fi.get());
+      if (!doRun(fn, ordered, skipPhi))
+         return false;
+   }
+   return !err;
+}
+
+bool
+Pass::run(Function *func, bool ordered, bool skipPhi)
+{
+   prog = func->getProgram();
+   err = false;
+   return doRun(func, ordered, skipPhi);
+}
+
+bool
+Pass::doRun(Function *func, bool ordered, bool skipPhi)
+{
+   Iterator *bbIter;
+   BasicBlock *bb;
+   Instruction *insn, *next;
+
+   this->func = func;
+   if (!visit(func))
+      return false;
+
+   bbIter = ordered ? func->cfg.iteratorCFG() : func->cfg.iteratorDFS();
+
+   for (; !bbIter->end(); bbIter->next()) {
+      bb = BasicBlock::get(reinterpret_cast<Graph::Node *>(bbIter->get()));
+      if (!visit(bb))
+         break;
+      for (insn = skipPhi ? bb->getEntry() : bb->getFirst(); insn != NULL;
+           insn = next) {
+         next = insn->next;
+         if (!visit(insn))
+            break;
+      }
+   }
+   func->cfg.putIterator(bbIter);
+   return !err;
+}
+
+void
+Function::printCFGraph(const char *filePath)
+{
+   FILE *out = fopen(filePath, "a");
+   if (!out) {
+      ERROR("failed to open file: %s\n", filePath);
+      return;
+   }
+   INFO("printing control flow graph to: %s\n", filePath);
+
+   fprintf(out, "digraph G {\n");
+
+   Iterator *iter;
+   for (iter = cfg.iteratorDFS(); !iter->end(); iter->next()) {
+      BasicBlock *bb = BasicBlock::get(
+         reinterpret_cast<Graph::Node *>(iter->get()));
+      int idA = bb->getId();
+      for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
+         int idB = BasicBlock::get(ei.getNode())->getId();
+         switch (ei.getType()) {
+         case Graph::Edge::TREE:
+            fprintf(out, "\t%i -> %i;\n", idA, idB);
+            break;
+         case Graph::Edge::FORWARD:
+            fprintf(out, "\t%i -> %i [color=green];\n", idA, idB);
+            break;
+         case Graph::Edge::CROSS:
+            fprintf(out, "\t%i -> %i [color=red];\n", idA, idB);
+            break;
+         case Graph::Edge::BACK:
+            fprintf(out, "\t%i -> %i;\n", idA, idB);
+            break;
+         case Graph::Edge::DUMMY:
+            fprintf(out, "\t%i -> %i [style=dotted];\n", idA, idB);
+            break;
+         default:
+            assert(0);
+            break;
+         }
+      }
+   }
+   cfg.putIterator(iter);
+
+   fprintf(out, "}\n");
+   fclose(out);
+}
+
+} // namespace nv50_ir
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_build_util.cpp b/src/gallium/drivers/nv50/codegen/nv50_ir_build_util.cpp
new file mode 100644
index 00000000000..284736838ab
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_build_util.cpp
@@ -0,0 +1,501 @@
+
+#include "nv50_ir.h"
+#include "nv50_ir_build_util.h"
+
+namespace nv50_ir {
+
+BuildUtil::BuildUtil()
+{
+   prog = NULL;
+   func = NULL;
+   bb = NULL;
+   pos = NULL;
+
+   memset(imms, 0, sizeof(imms));
+   immCount = 0;
+}
+
+void
+BuildUtil::addImmediate(ImmediateValue *imm)
+{
+   if (immCount > (NV50_IR_BUILD_IMM_HT_SIZE * 3) / 4)
+      return;
+
+   unsigned int pos = u32Hash(imm->reg.data.u32);
+
+   while (imms[pos])
+      pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE;
+   imms[pos] = imm;
+   immCount++;
+}
+
+Instruction *
+BuildUtil::mkOp1(operation op, DataType ty, Value *dst, Value *src)
+{
+   Instruction *insn = new_Instruction(func, op, ty);
+
+   insn->setDef(0, dst);
+   insn->setSrc(0, src);
+
+   insert(insn);
+   return insn;
+}
+
+Instruction *
+BuildUtil::mkOp2(operation op, DataType ty, Value *dst,
+                 Value *src0, Value *src1)
+{
+   Instruction *insn = new_Instruction(func, op, ty);
+
+   insn->setDef(0, dst);
+   insn->setSrc(0, src0);
+   insn->setSrc(1, src1);
+
+   insert(insn);
+   return insn;
+}
+
+Instruction *
+BuildUtil::mkOp3(operation op, DataType ty, Value *dst,
+                 Value *src0, Value *src1, Value *src2)
+{
+   Instruction *insn = new_Instruction(func, op, ty);
+
+   insn->setDef(0, dst);
+   insn->setSrc(0, src0);
+   insn->setSrc(1, src1);
+   insn->setSrc(2, src2);
+
+   insert(insn);
+   return insn;
+}
+
+LValue *
+BuildUtil::mkLoad(DataType ty, Symbol *mem, Value *ptr)
+{
+   Instruction *insn = new_Instruction(func, OP_LOAD, ty);
+   LValue *def = getScratch();
+
+   insn->setDef(0, def);
+   insn->setSrc(0, mem);
+   if (ptr)
+      insn->setIndirect(0, 0, ptr);
+
+   insert(insn);
+   return def;
+}
+
+Instruction *
+BuildUtil::mkStore(operation op, DataType ty, Symbol *mem, Value *ptr,
+                   Value *stVal)
+{
+   Instruction *insn = new_Instruction(func, op, ty);
+
+   insn->setSrc(0, mem);
+   insn->setSrc(1, stVal);
+   if (ptr)
+      insn->setIndirect(0, 0, ptr);
+
+   insert(insn);
+   return insn;
+}
+
+Instruction *
+BuildUtil::mkFetch(Value *dst, DataType ty, DataFile file, int32_t offset,
+                   Value *attrRel, Value *primRel)
+{
+   Symbol *sym = mkSymbol(file, 0, ty, offset);
+
+   Instruction *insn = mkOp1(OP_VFETCH, ty, dst, sym);
+
+   insn->setIndirect(0, 0, attrRel);
+   insn->setIndirect(0, 1, primRel);
+
+   // already inserted
+   return insn;
+}
+
+Instruction *
+BuildUtil::mkMov(Value *dst, Value *src, DataType ty)
+{
+   Instruction *insn = new_Instruction(func, OP_MOV, ty);
+
+   insn->setDef(0, dst);
+   insn->setSrc(0, src);
+
+   insert(insn);
+   return insn;
+}
+
+Instruction *
+BuildUtil::mkMovToReg(int id, Value *src)
+{
+   Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(src->reg.size));
+
+   insn->setDef(0, new_LValue(func, FILE_GPR));
+   insn->getDef(0)->reg.data.id = id;
+   insn->setSrc(0, src);
+
+   insert(insn);
+   return insn;
+}
+
+Instruction *
+BuildUtil::mkMovFromReg(Value *dst, int id)
+{
+   Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(dst->reg.size));
+
+   insn->setDef(0, dst);
+   insn->setSrc(0, new_LValue(func, FILE_GPR));
+   insn->getSrc(0)->reg.data.id = id;
+
+   insert(insn);
+   return insn;
+}
+
+Instruction *
+BuildUtil::mkCvt(operation op,
+                 DataType dstTy, Value *dst, DataType srcTy, Value *src)
+{
+   Instruction *insn = new_Instruction(func, op, dstTy);
+
+   insn->setType(dstTy, srcTy);
+   insn->setDef(0, dst);
+   insn->setSrc(0, src);
+
+   insert(insn);
+   return insn;
+}
+
+Instruction *
+BuildUtil::mkCmp(operation op, CondCode cc, DataType ty, Value *dst,
+                 Value *src0, Value *src1, Value *src2)
+{
+   CmpInstruction *insn = new_CmpInstruction(func, op);
+
+   insn->setType(dst->reg.file == FILE_PREDICATE ? TYPE_U8 : ty, ty);
+   insn->setCondition(cc);
+   insn->setDef(0, dst);
+   insn->setSrc(0, src0);
+   insn->setSrc(1, src1);
+   if (src2)
+      insn->setSrc(2, src2);
+
+   insert(insn);
+   return insn;
+}
+
+Instruction *
+BuildUtil::mkTex(operation op, TexTarget targ, uint8_t tic, uint8_t tsc,
+                 Value **def, Value **src)
+{
+   TexInstruction *tex = new_TexInstruction(func, op);
+
+   for (int d = 0; d < 4 && def[d]; ++d)
+      tex->setDef(d, def[d]);
+   for (int s = 0; s < 4 && src[s]; ++s)
+      tex->setSrc(s, src[s]);
+
+   tex->setTexture(targ, tic, tsc);
+
+   return tex;
+}
+
+Instruction *
+BuildUtil::mkQuadop(uint8_t q, Value *def, uint8_t l, Value *src0, Value *src1)
+{
+   Instruction *quadop = mkOp2(OP_QUADOP, TYPE_F32, def, src0, src1);
+   quadop->subOp = q;
+   quadop->lanes = l;
+   return quadop;
+}
+
+Instruction *
+BuildUtil::mkSelect(Value *pred, Value *dst, Value *trSrc, Value *flSrc)
+{
+   Instruction *insn;
+   LValue *def0 = getSSA();
+   LValue *def1 = getSSA();
+
+   mkMov(def0, trSrc)->setPredicate(CC_P, pred);
+   mkMov(def1, flSrc)->setPredicate(CC_NOT_P, pred);
+
+   insn = mkOp2(OP_UNION, typeOfSize(dst->reg.size), dst, def0, def1);
+
+   insert(insn);
+   return insn;
+}
+
+FlowInstruction *
+BuildUtil::mkFlow(operation op, BasicBlock *targ, CondCode cc, Value *pred)
+{
+   FlowInstruction *insn = new_FlowInstruction(func, op, targ);
+
+   if (pred)
+      insn->setPredicate(cc, pred);
+
+   insert(insn);
+   return insn;
+}
+
+void
+BuildUtil::mkClobber(DataFile f, uint32_t rMask, int unit)
+{
+   static const uint16_t baseSize2[16] =
+   {
+      0x0000, 0x0010, 0x0011, 0x0020, 0x0012, 0x1210, 0x1211, 0x1220,
+      0x0013, 0x1310, 0x1311, 0x0020, 0x1320, 0x0022, 0x2210, 0x0040,
+   };
+
+   int base = 0;
+
+   for (; rMask; rMask >>= 4, base += 4) {
+      const uint32_t mask = rMask & 0xf;
+      if (!mask)
+         continue;
+      int base1 = (baseSize2[mask] >>  0) & 0xf;
+      int size1 = (baseSize2[mask] >>  4) & 0xf;
+      int base2 = (baseSize2[mask] >>  8) & 0xf;
+      int size2 = (baseSize2[mask] >> 12) & 0xf;
+      Instruction *insn = mkOp(OP_NOP, TYPE_NONE, NULL);
+      if (1) { // size1 can't be 0
+         LValue *reg = new_LValue(func, f);
+         reg->reg.size = size1 << unit;
+         reg->reg.data.id = base + base1;
+         insn->setDef(0, reg);
+      }
+      if (size2) {
+         LValue *reg = new_LValue(func, f);
+         reg->reg.size = size2 << unit;
+         reg->reg.data.id = base + base2;
+         insn->setDef(1, reg);
+      }
+   }
+}
+
+ImmediateValue *
+BuildUtil::mkImm(uint32_t u)
+{
+   unsigned int pos = u32Hash(u);
+
+   while (imms[pos] && imms[pos]->reg.data.u32 != u)
+      pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE;
+
+   ImmediateValue *imm = imms[pos];
+   if (!imm) {
+      imm = new_ImmediateValue(prog, u);
+      addImmediate(imm);
+   }
+   return imm;
+}
+
+ImmediateValue *
+BuildUtil::mkImm(uint64_t u)
+{
+   ImmediateValue *imm = new_ImmediateValue(prog, (uint32_t)0);
+
+   imm->reg.size = 8;
+   imm->reg.type = TYPE_U64;
+   imm->reg.data.u64 = u;
+
+   return imm;
+}
+
+ImmediateValue *
+BuildUtil::mkImm(float f)
+{
+   union {
+      float f32;
+      uint32_t u32;
+   } u;
+   u.f32 = f;
+   return mkImm(u.u32);
+}
+
+Value *
+BuildUtil::loadImm(Value *dst, float f)
+{
+   return mkOp1v(OP_MOV, TYPE_F32, dst ? dst : getScratch(), mkImm(f));
+}
+
+Value *
+BuildUtil::loadImm(Value *dst, uint32_t u)
+{
+   return mkOp1v(OP_MOV, TYPE_U32, dst ? dst : getScratch(), mkImm(u));
+}
+
+Value *
+BuildUtil::loadImm(Value *dst, uint64_t u)
+{
+   return mkOp1v(OP_MOV, TYPE_U64, dst ? dst : getScratch(8), mkImm(u));
+}
+
+Symbol *
+BuildUtil::mkSymbol(DataFile file, int8_t fileIndex, DataType ty,
+                    uint32_t baseAddr)
+{
+   Symbol *sym = new_Symbol(prog, file, fileIndex);
+
+   sym->setOffset(baseAddr);
+   sym->reg.type = ty;
+   sym->reg.size = typeSizeof(ty);
+
+   return sym;
+}
+
+Symbol *
+BuildUtil::mkSysVal(SVSemantic svName, uint32_t svIndex)
+{
+   Symbol *sym = new_Symbol(prog, FILE_SYSTEM_VALUE, 0);
+
+   assert(svIndex < 4 ||
+          (svName == SV_CLIP_DISTANCE || svName == SV_TESS_FACTOR));
+
+   switch (svName) {
+   case SV_POSITION:
+   case SV_FACE:
+   case SV_YDIR:
+   case SV_POINT_SIZE:
+   case SV_POINT_COORD:
+   case SV_CLIP_DISTANCE:
+   case SV_TESS_FACTOR:
+      sym->reg.type = TYPE_F32;
+      break;
+   default:
+      sym->reg.type = TYPE_U32;
+      break;
+   }
+   sym->reg.size = typeSizeof(sym->reg.type);
+
+   sym->reg.data.sv.sv = svName;
+   sym->reg.data.sv.index = svIndex;
+
+   return sym;
+}
+
+void
+BuildUtil::DataArray::init()
+{
+   values = NULL;
+   baseAddr = 0;
+   arrayLen = 0;
+
+   vecDim = 4;
+   eltSize = 2;
+
+   file = FILE_GPR;
+   regOnly = true;
+}
+
+BuildUtil::DataArray::DataArray()
+{
+   init();
+}
+
+BuildUtil::DataArray::DataArray(BuildUtil *bld) : up(bld)
+{
+   init();
+}
+
+BuildUtil::DataArray::~DataArray()
+{
+   if (values)
+      delete[] values;
+}
+
+void
+BuildUtil::DataArray::setup(uint32_t base, int len, int v, int size,
+                            DataFile f, int8_t fileIndex)
+{
+   baseAddr = base;
+   arrayLen = len;
+
+   vecDim = v;
+   eltSize = size;
+
+   file = f;
+   regOnly = !isMemoryFile(f);
+
+   values = new Value * [arrayLen * vecDim];
+   if (values)
+      memset(values, 0, arrayLen * vecDim * sizeof(Value *));
+
+   if (!regOnly) {
+      baseSym = new_Symbol(up->getProgram(), file, fileIndex);
+      baseSym->setOffset(baseAddr);
+      baseSym->reg.size = size;
+   }
+}
+
+Value *
+BuildUtil::DataArray::acquire(int i, int c)
+{
+   const unsigned int idx = i * vecDim + c;
+
+   assert(idx < arrayLen * vecDim);
+
+   if (regOnly) {
+      const unsigned int idx = i * 4 + c; // vecDim always 4 if regOnly
+      if (!values[idx])
+         values[idx] = new_LValue(up->getFunction(), file);
+      return values[idx];
+   } else {
+      return up->getScratch();
+   }
+}
+
+Value *
+BuildUtil::DataArray::load(int i, int c, Value *ptr)
+{
+   const unsigned int idx = i * vecDim + c;
+
+   assert(idx < arrayLen * vecDim);
+
+   if (regOnly) {
+      if (!values[idx])
+         values[idx] = new_LValue(up->getFunction(), file);
+      return values[idx];
+   } else {
+      Symbol *sym = reinterpret_cast<Symbol *>(values[idx]);
+      if (!sym)
+         values[idx] = sym = this->mkSymbol(i, c, baseSym);
+      return up->mkLoad(typeOfSize(eltSize), sym, ptr);
+   }
+}
+
+void
+BuildUtil::DataArray::store(int i, int c, Value *ptr, Value *value)
+{
+   const unsigned int idx = i * vecDim + c;
+
+   assert(idx < arrayLen * vecDim);
+
+   if (regOnly) {
+      assert(!ptr);
+      assert(!values[idx] || values[idx] == value);
+      values[idx] = value;
+   } else {
+      Symbol *sym = reinterpret_cast<Symbol *>(values[idx]);
+      if (!sym)
+         values[idx] = sym = this->mkSymbol(i, c, baseSym);
+      up->mkStore(OP_STORE, typeOfSize(value->reg.size), sym, ptr, value);
+   }
+}
+
+Symbol *
+BuildUtil::DataArray::mkSymbol(int i, int c, Symbol *base)
+{
+   const unsigned int idx = i * vecDim + c;
+
+   Symbol *sym = new_Symbol(up->getProgram(), file, 0);
+
+   assert(base || (idx < arrayLen && c < vecDim));
+
+   sym->reg.size = eltSize;
+   sym->reg.type = typeOfSize(eltSize);
+
+   sym->setAddress(base, baseAddr + idx * eltSize);
+   return sym;
+}
+
+} // namespace nv50_ir
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_build_util.h b/src/gallium/drivers/nv50/codegen/nv50_ir_build_util.h
new file mode 100644
index 00000000000..4c3addb27e4
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_build_util.h
@@ -0,0 +1,245 @@
+
+#ifndef __NV50_IR_BUILD_UTIL__
+#define __NV50_IR_BUILD_UTIL__
+
+namespace nv50_ir {
+
+class BuildUtil
+{
+public:
+   BuildUtil();
+
+   inline void setProgram(Program *);
+   inline Program *getProgram() const { return prog; }
+   inline Function *getFunction() const { return func; }
+
+   // keeps inserting at head/tail of block
+   inline void setPosition(BasicBlock *, bool tail);
+   // position advances only if @after is true
+   inline void setPosition(Instruction *, bool after);
+
+   inline BasicBlock *getBB() { return bb; }
+
+   inline void insert(Instruction *);
+   inline void remove(Instruction *i) { assert(i->bb == bb); bb->remove(i); }
+
+   inline LValue *getScratch(int size = 4);
+   inline LValue *getSSA(int size = 4); // scratch value for a single assignment
+
+   inline Instruction *mkOp(operation, DataType, Value *);
+   Instruction *mkOp1(operation, DataType, Value *, Value *);
+   Instruction *mkOp2(operation, DataType, Value *, Value *, Value *);
+   Instruction *mkOp3(operation, DataType, Value *, Value *, Value *, Value *);
+
+   LValue *mkOp1v(operation, DataType, Value *, Value *);
+   LValue *mkOp2v(operation, DataType, Value *, Value *, Value *);
+   LValue *mkOp3v(operation, DataType, Value *, Value *, Value *, Value *);
+
+   LValue *mkLoad(DataType, Symbol *, Value *ptr);
+   Instruction *mkStore(operation, DataType, Symbol *, Value *ptr, Value *val);
+
+   Instruction *mkMov(Value *, Value *, DataType = TYPE_U32);
+   Instruction *mkMovToReg(int id, Value *);
+   Instruction *mkMovFromReg(Value *, int id);
+
+   Instruction *mkFetch(Value *, DataType, DataFile, int32_t offset,
+                        Value *attrRel, Value *primRel);
+
+   Instruction *mkCvt(operation, DataType, Value *, DataType, Value *);
+   Instruction *mkCmp(operation, CondCode, DataType,
+                      Value *,
+                      Value *, Value *, Value * = NULL);
+   Instruction *mkTex(operation, TexTarget, uint8_t tic, uint8_t tsc,
+                      Value **def, Value **src);
+   Instruction *mkQuadop(uint8_t qop, Value *, uint8_t l, Value *, Value *);
+
+   FlowInstruction *mkFlow(operation, BasicBlock *target,
+                           CondCode, Value *pred);
+
+   Instruction *mkSelect(Value *pred, Value *dst, Value *trSrc, Value *flSrc);
+
+   void mkClobber(DataFile file, uint32_t regMask, int regUnitLog2);
+
+   ImmediateValue *mkImm(float);
+   ImmediateValue *mkImm(uint32_t);
+   ImmediateValue *mkImm(uint64_t);
+
+   ImmediateValue *mkImm(int i) { return mkImm((uint32_t)i); }
+
+   Value *loadImm(Value *dst, float);
+   Value *loadImm(Value *dst, uint32_t);
+   Value *loadImm(Value *dst, uint64_t);
+
+   Value *loadImm(Value *dst, int i) { return loadImm(dst, (uint32_t)i); }
+
+   class DataArray
+   {
+   public:
+      DataArray();
+      DataArray(BuildUtil *);
+      ~DataArray();
+
+      inline void setParent(BuildUtil *bld) { assert(!up); up = bld; }
+
+      void setup(uint32_t base, int len, int vecDim, int size,
+                 DataFile, int8_t fileIndex = 0);
+
+      inline bool exists(unsigned int i, unsigned int c);
+
+      Value *load(int i, int c, Value *ptr);
+      void store(int i, int c, Value *ptr, Value *value);
+      Value *acquire(int i, int c);
+
+   private:
+      Symbol *mkSymbol(int i, int c, Symbol *base);
+
+   private:
+      Value **values;
+      uint32_t baseAddr;
+      uint32_t arrayLen;
+      Symbol *baseSym;
+
+      uint8_t vecDim;
+      uint8_t eltSize; // in bytes
+
+      DataFile file;
+      bool regOnly;
+
+      BuildUtil *up;
+
+      void init();
+   };
+
+   Symbol *mkSymbol(DataFile file, int8_t fileIndex,
+                    DataType ty, uint32_t baseAddress);
+
+   Symbol *mkSysVal(SVSemantic svName, uint32_t svIndex);
+
+private:
+   void addImmediate(ImmediateValue *);
+   inline unsigned int u32Hash(uint32_t);
+
+protected:
+   Program *prog;
+   Function *func;
+   Instruction *pos;
+   BasicBlock *bb;
+   bool tail;
+
+#define NV50_IR_BUILD_IMM_HT_SIZE 256
+
+   ImmediateValue *imms[NV50_IR_BUILD_IMM_HT_SIZE];
+   unsigned int immCount;
+};
+
+unsigned int BuildUtil::u32Hash(uint32_t u)
+{
+   return (u % 273) % NV50_IR_BUILD_IMM_HT_SIZE;
+}
+
+void BuildUtil::setProgram(Program *program)
+{
+   prog = program;
+}
+
+void
+BuildUtil::setPosition(BasicBlock *block, bool atTail)
+{
+   bb = block;
+   prog = bb->getProgram();
+   func = bb->getFunction();
+   pos = NULL;
+   tail = atTail;
+}
+
+void
+BuildUtil::setPosition(Instruction *i, bool after)
+{
+   bb = i->bb;
+   prog = bb->getProgram();
+   func = bb->getFunction();
+   pos = i;
+   tail = after;
+   assert(bb);
+}
+
+LValue *
+BuildUtil::getScratch(int size)
+{
+   LValue *lval = new_LValue(func, FILE_GPR);
+   if (size != 4)
+      lval->reg.size = size;
+   return lval;
+}
+
+LValue *
+BuildUtil::getSSA(int size)
+{
+   LValue *lval = new_LValue(func, FILE_GPR);
+   lval->ssa = 1;
+   if (size != 4)
+      lval->reg.size = size;
+   return lval;
+}
+
+void BuildUtil::insert(Instruction *i)
+{
+   if (!pos) {
+      tail ? bb->insertTail(i) : bb->insertHead(i);
+   } else {
+      if (tail) {
+         bb->insertAfter(pos, i);
+         pos = i;
+      } else {
+         bb->insertBefore(pos, i);
+      }
+   }
+}
+
+Instruction *
+BuildUtil::mkOp(operation op, DataType ty, Value *dst)
+{
+   Instruction *insn = new_Instruction(func, op, ty);
+   insn->setDef(0, dst);
+   insert(insn);
+   if (op == OP_DISCARD || op == OP_EXIT ||
+       op == OP_JOIN ||
+       op == OP_QUADON || op == OP_QUADPOP ||
+       op == OP_EMIT || op == OP_RESTART)
+      insn->fixed = 1;
+   return insn;
+}
+
+inline LValue *
+BuildUtil::mkOp1v(operation op, DataType ty, Value *dst, Value *src)
+{
+   mkOp1(op, ty, dst, src);
+   return dst->asLValue();
+}
+
+inline LValue *
+BuildUtil::mkOp2v(operation op, DataType ty, Value *dst,
+                  Value *src0, Value *src1)
+{
+   mkOp2(op, ty, dst, src0, src1);
+   return dst->asLValue();
+}
+
+inline LValue *
+BuildUtil::mkOp3v(operation op, DataType ty, Value *dst,
+                  Value *src0, Value *src1, Value *src2)
+{
+   mkOp3(op, ty, dst, src0, src1, src2);
+   return dst->asLValue();
+}
+
+bool
+BuildUtil::DataArray::exists(unsigned int i, unsigned int c)
+{
+   assert(i < arrayLen && c < vecDim);
+   return !regOnly || values[i * vecDim + c];
+}
+
+} // namespace nv50_ir
+
+#endif // __NV50_IR_BUILD_UTIL_H__
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_driver.h b/src/gallium/drivers/nv50/codegen/nv50_ir_driver.h
new file mode 100644
index 00000000000..27e435d4ea1
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_driver.h
@@ -0,0 +1,149 @@
+
+#ifndef __NV50_IR_DRIVER_H__
+#define __NV50_IR_DRIVER_H__
+
+#include "pipe/p_shader_tokens.h"
+
+#include "tgsi/tgsi_util.h"
+#include "tgsi/tgsi_parse.h"
+#include "tgsi/tgsi_scan.h"
+
+/*
+ * This struct constitutes linkage information in TGSI terminology.
+ *
+ * It is created by the code generator and handed to the pipe driver
+ * for input/output slot assignment.
+ */
+struct nv50_ir_varying
+{
+   uint8_t slot[4]; /* native slots for xyzw (addresses in 32-bit words) */
+
+   unsigned mask     : 4; /* vec4 mask */
+   unsigned linear   : 1; /* linearly interpolated if true (and not flat) */
+   unsigned flat     : 1;
+   unsigned centroid : 1;
+   unsigned patch    : 1; /* patch constant value */
+   unsigned regular  : 1; /* driver-specific meaning (e.g. input in sreg) */
+   unsigned input    : 1; /* indicates direction of system values */
+   unsigned oread    : 1; /* true if output is read from parallel TCP */
+
+   ubyte id; /* TGSI register index */
+   ubyte sn; /* TGSI semantic name */
+   ubyte si; /* TGSI semantic index */
+};
+
+#define NV50_PROGRAM_IR_TGSI 0
+#define NV50_PROGRAM_IR_SM4  1
+#define NV50_PROGRAM_IR_GLSL 2
+#define NV50_PROGRAM_IR_LLVM 3
+
+#ifdef DEBUG
+# define NV50_IR_DEBUG_BASIC     (1 << 0)
+# define NV50_IR_DEBUG_VERBOSE   (2 << 0)
+# define NV50_IR_DEBUG_REG_ALLOC (1 << 2)
+#else
+# define NV50_IR_DEBUG_BASIC     0
+# define NV50_IR_DEBUG_VERBOSE   0
+# define NV50_IR_DEBUG_REG_ALLOC 0
+#endif
+
+struct nv50_ir_prog_info
+{
+   uint16_t target; /* chipset (0x50, 0x84, 0xc0, ...) */
+
+   uint8_t type; /* PIPE_SHADER */
+
+   uint8_t optLevel; /* optimization level (0 to 3) */
+   uint8_t dbgFlags;
+
+   struct {
+      int16_t maxGPR;     /* may be -1 if none used */
+      int16_t maxOutput;
+      uint32_t tlsSpace;  /* required local memory per thread */
+      uint32_t *code;
+      uint32_t codeSize;
+      uint8_t sourceRep;  /* NV50_PROGRAM_IR */
+      const void *source;
+      void *relocData;
+   } bin;
+
+   struct nv50_ir_varying sv[PIPE_MAX_SHADER_INPUTS];
+   struct nv50_ir_varying in[PIPE_MAX_SHADER_INPUTS];
+   struct nv50_ir_varying out[PIPE_MAX_SHADER_OUTPUTS];
+   uint8_t numInputs;
+   uint8_t numOutputs;
+   uint8_t numPatchConstants; /* also included in numInputs/numOutputs */
+   uint8_t numSysVals;
+
+   struct {
+      uint32_t *buf;    /* for IMMEDIATE_ARRAY */
+      uint16_t bufSize; /* size of immediate array */
+      uint16_t count;   /* count of inline immediates */
+      uint32_t *data;   /* inline immediate data */
+      uint8_t *type;    /* for each vec4 (128 bit) */
+   } immd;
+
+   union {
+      struct {
+         uint32_t inputMask[4]; /* mask of attributes read (1 bit per scalar) */
+      } vp;
+      struct {
+         uint8_t inputPatchSize;
+         uint8_t outputPatchSize;
+         uint8_t partitioning;    /* PIPE_TESS_PART */
+         int8_t winding;          /* +1 (clockwise) / -1 (counter-clockwise) */
+         uint8_t domain;          /* PIPE_PRIM_{QUADS,TRIANGLES,LINES} */
+         uint8_t outputPrim;      /* PIPE_PRIM_{TRIANGLES,LINES,POINTS} */
+      } tp;
+      struct {
+         uint8_t inputPrim;
+         uint8_t outputPrim;
+         unsigned instanceCount;
+         unsigned maxVertices;
+      } gp;
+      struct {
+         unsigned numColourResults;
+         boolean writesDepth;
+         boolean earlyFragTests;
+         boolean separateFragData;
+         boolean usesDiscard;
+      } fp;
+   } prop;
+
+   struct {
+      uint8_t clipDistance;      /* index of first clip distance output */
+      uint8_t clipDistanceCount;
+      uint8_t cullDistanceMask;  /* clip distance mode (1 bit per output) */
+      uint8_t pointSize;         /* output index for PointSize */
+      uint8_t edgeFlagIn;
+      uint8_t edgeFlagOut;
+      uint8_t fragDepth;         /* output index of FragDepth */
+      uint8_t sampleMask;        /* output index of SampleMask */
+      uint8_t backFaceColor[2];  /* input/output indices of back face colour */
+      uint8_t globalAccess;      /* 1 for read, 2 for wr, 3 for rw */
+   } io;
+
+   /* driver callback to assign input/output locations */
+   int (*assignSlots)(struct nv50_ir_prog_info *);
+};
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int nv50_ir_generate_code(struct nv50_ir_prog_info *);
+
+extern void nv50_ir_relocate_code(void *relocData, uint32_t *code,
+                                  uint32_t codePos,
+                                  uint32_t libPos,
+                                  uint32_t dataPos);
+
+/* obtain code that will be shared among programs */
+extern void nv50_ir_get_target_library(uint32_t chipset,
+                                       const uint32_t **code, uint32_t *size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // __NV50_IR_DRIVER_H__
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_emit_nv50.cpp b/src/gallium/drivers/nv50/codegen/nv50_ir_emit_nv50.cpp
new file mode 100644
index 00000000000..0a61a1ddaef
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_emit_nv50.cpp
@@ -0,0 +1,1333 @@
+
+#include "nv50_ir.h"
+#include "nv50_ir_target.h"
+
+namespace nv50_ir {
+
+class CodeEmitterNV50 : public CodeEmitter
+{
+public:
+   CodeEmitterNV50(const Target *);
+
+   virtual bool emitInstruction(Instruction *);
+
+   virtual uint32_t getMinEncodingSize(const Instruction *) const;
+
+   inline void setProgramType(Program::Type pType) { progType = pType; }
+
+private:
+   const Target *targ;
+
+   Program::Type progType;
+
+private:
+   inline void defId(const ValueDef&, const int pos);
+   inline void srcId(const ValueRef&, const int pos);
+   inline void srcId(const ValueRef *, const int pos);
+
+   inline void srcAddr16(const ValueRef&, const int pos);
+   inline void srcAddr8(const ValueRef&, const int pos);
+
+   void emitFlagsRd(const Instruction *);
+   void emitFlagsWr(const Instruction *);
+
+   void emitCondCode(CondCode cc, int pos);
+
+   inline void setARegBits(unsigned int);
+
+   void setAReg16(const Instruction *, int s);
+   void setImmediate(const Instruction *, int s);
+
+   void setDst(const Value *);
+   void setDst(const Instruction *, int d);
+   void emitSrc0(const ValueRef&);
+   void emitSrc1(const ValueRef&);
+   void emitSrc2(const ValueRef&);
+
+   void emitForm_MAD(const Instruction *);
+   void emitForm_ADD(const Instruction *);
+   void emitForm_MUL(const Instruction *);
+   void emitForm_IMM(const Instruction *);
+
+   void emitLoadStoreSize(DataType ty, int pos);
+
+   void roundMode_MAD(const Instruction *);
+   void roundMode_CVT(RoundMode);
+
+   void emitMNeg12(const Instruction *);
+
+   void emitLOAD(const Instruction *);
+   void emitSTORE(const Instruction *);
+   void emitMOV(const Instruction *);
+   void emitNOP();
+   void emitINTERP(const Instruction *);
+   void emitPFETCH(const Instruction *);
+   void emitOUT(const Instruction *);
+
+   void emitUADD(const Instruction *);
+   void emitAADD(const Instruction *);
+   void emitFADD(const Instruction *);
+   void emitUMUL(const Instruction *);
+   void emitFMUL(const Instruction *);
+   void emitFMAD(const Instruction *);
+
+   void emitMINMAX(const Instruction *);
+
+   void emitPreOp(const Instruction *);
+   void emitSFnOp(const Instruction *, uint8_t subOp);
+
+   void emitShift(const Instruction *);
+   void emitARL(const Instruction *);
+   void emitLogicOp(const Instruction *);
+
+   void emitCVT(const Instruction *);
+   void emitSET(const Instruction *);
+
+   void emitTEX(const TexInstruction *);
+
+   void emitQUADOP(const Instruction *, uint8_t lane, uint8_t quOp);
+
+   void emitFlow(const Instruction *, uint8_t flowOp);
+};
+
+#define SDATA(a) ((a).rep()->reg.data)
+#define DDATA(a) ((a).rep()->reg.data)
+
+void CodeEmitterNV50::srcId(const ValueRef& src, const int pos)
+{
+   assert(src.get());
+   code[pos / 32] |= SDATA(src).id << (pos % 32);
+}
+
+void CodeEmitterNV50::srcId(const ValueRef *src, const int pos)
+{
+   assert(src->get());
+   code[pos / 32] |= SDATA(*src).id << (pos % 32);
+}
+
+void CodeEmitterNV50::srcAddr16(const ValueRef& src, const int pos)
+{
+   assert(src.get());
+
+   uint32_t offset = SDATA(src).offset;
+
+   assert(offset <= 0xffff && (pos % 32) <= 16);
+
+   code[pos / 32] |= offset << (pos % 32);
+}
+
+void CodeEmitterNV50::srcAddr8(const ValueRef& src, const int pos)
+{
+   assert(src.get());
+
+   uint32_t offset = SDATA(src).offset;
+
+   assert(offset <= 0x1fc && !(offset & 0x3));
+
+   code[pos / 32] |= (offset >> 2) << (pos % 32);
+}
+
+void CodeEmitterNV50::defId(const ValueDef& def, const int pos)
+{
+   assert(def.get());
+   code[pos / 32] |= DDATA(def).id << (pos % 32);
+}
+
+void
+CodeEmitterNV50::roundMode_MAD(const Instruction *insn)
+{
+   switch (insn->rnd) {
+   case ROUND_M: code[1] |= 1 << 22; break;
+   case ROUND_P: code[1] |= 2 << 22; break;
+   case ROUND_Z: code[1] |= 3 << 22; break;
+   default:
+      assert(insn->rnd == ROUND_N);
+      break;
+   }
+}
+
+void
+CodeEmitterNV50::emitMNeg12(const Instruction *i)
+{
+   code[1] |= i->src[0].mod.neg() << 26;
+   code[1] |= i->src[1].mod.neg() << 27;
+}
+
+void CodeEmitterNV50::emitCondCode(CondCode cc, int pos)
+{
+   uint8_t enc;
+
+   assert(pos >= 32 || pos <= 27);
+
+   switch (cc) {
+   case CC_LT:  enc = 0x1; break;
+   case CC_LTU: enc = 0x9; break;
+   case CC_EQ:  enc = 0x2; break;
+   case CC_EQU: enc = 0xa; break;
+   case CC_LE:  enc = 0x3; break;
+   case CC_LEU: enc = 0xb; break;
+   case CC_GT:  enc = 0x4; break;
+   case CC_GTU: enc = 0xc; break;
+   case CC_NE:  enc = 0x5; break;
+   case CC_NEU: enc = 0xd; break;
+   case CC_GE:  enc = 0x6; break;
+   case CC_GEU: enc = 0xe; break;
+   case CC_TR:  enc = 0xf; break;
+   case CC_FL:  enc = 0x0; break;
+
+   case CC_O:  enc = 0x10; break;
+   case CC_C:  enc = 0x11; break;
+   case CC_A:  enc = 0x12; break;
+   case CC_S:  enc = 0x13; break;
+   case CC_NS: enc = 0x1c; break;
+   case CC_NA: enc = 0x1d; break;
+   case CC_NC: enc = 0x1e; break;
+   case CC_NO: enc = 0x1f; break;
+
+   default:
+      enc = 0;
+      assert(!"invalid condition code");
+      break;
+   }
+   code[pos / 32] |= enc << (pos % 32);
+}
+
+void
+CodeEmitterNV50::emitFlagsRd(const Instruction *i)
+{
+   int s = (i->flagsSrc >= 0) ? i->flagsSrc : i->predSrc;
+
+   assert(!(code[1] & 0x00003f80));
+
+   if (s >= 0) {
+      assert(i->getSrc(s)->reg.file == FILE_FLAGS);
+      emitCondCode(i->cc, 32 + 7);
+      srcId(i->src[s], 32 + 12);
+   } else {
+      code[1] |= 0x0780;
+   }
+}
+
+void
+CodeEmitterNV50::emitFlagsWr(const Instruction *i)
+{
+   assert(!(code[1] & 0x70));
+
+   if (i->flagsDef >= 0)
+      code[1] |= (DDATA(i->def[i->flagsDef]).id << 4) | 0x40;
+}
+
+void
+CodeEmitterNV50::setARegBits(unsigned int u)
+{
+   code[0] |= (u & 3) << 26;
+   code[1] |= (u & 4);
+}
+
+void
+CodeEmitterNV50::setAReg16(const Instruction *i, int s)
+{
+   s = i->src[s].indirect[0];
+   if (s >= 0)
+      setARegBits(SDATA(i->src[s]).id + 1);
+}
+
+void
+CodeEmitterNV50::setImmediate(const Instruction *i, int s)
+{
+   const ImmediateValue *imm = i->src[s].get()->asImm();
+   assert(imm);
+
+   code[1] |= 3;
+   code[0] |= (imm->reg.data.u32 & 0x3f) << 16;
+   code[1] |= (imm->reg.data.u32 >> 6) << 2;
+}
+
+void
+CodeEmitterNV50::setDst(const Value *dst)
+{
+   const Storage *reg = &dst->join->reg;
+
+   assert(reg->file != FILE_ADDRESS);
+
+   if (reg->data.id < 0) {
+      code[0] |= (127 << 2) | 1;
+      code[1] |= 8;
+   } else {
+      if (reg->file == FILE_SHADER_OUTPUT)
+         code[1] |= 8;
+      code[0] |= reg->data.id << 2;
+   }
+}
+
+void
+CodeEmitterNV50::setDst(const Instruction *i, int d)
+{
+   if (i->defExists(d)) {
+      setDst(i->getDef(d));
+   } else
+   if (!d) {
+      code[0] |= 0x01fc; // bit bucket
+      code[1] |= 0x0008;
+   }
+}
+
+void
+CodeEmitterNV50::emitSrc0(const ValueRef& ref)
+{
+   const Storage *reg = &ref.rep()->reg;
+
+   if (reg->file == FILE_SHADER_INPUT)
+      code[1] |= 0x00200000;
+   else
+   if (reg->file != FILE_GPR)
+      ERROR("invalid src0 register file: %d\n", reg->file);
+
+   assert(reg->data.id < 128);
+   code[0] |= reg->data.id << 9;
+}
+
+void
+CodeEmitterNV50::emitSrc1(const ValueRef& ref)
+{
+   const Storage *reg = &ref.rep()->reg;
+
+   if (reg->file == FILE_MEMORY_CONST) {
+      assert(!(code[1] & 0x01800000));
+      code[0] |= 1 << 23;
+      code[1] |= reg->fileIndex << 22;
+   } else
+   if (reg->file != FILE_GPR) {
+      ERROR("invalid src1 register file: %d\n", reg->file);
+   }
+
+   assert(reg->data.id < 128);
+   code[0] |= reg->data.id << 16;
+}
+
+void
+CodeEmitterNV50::emitSrc2(const ValueRef& ref)
+{
+   const Storage *reg = &ref.rep()->reg;
+
+   if (reg->file == FILE_MEMORY_CONST) {
+      assert(!(code[1] & 0x01800000));
+      code[0] |= 1 << 24;
+      code[1] |= reg->fileIndex << 22;
+   } else
+   if (reg->file != FILE_GPR) {
+      ERROR("invalid src1 register file: %d\n", reg->file);
+   }
+
+   assert(reg->data.id < 128);
+   code[1] |= reg->data.id << 14;
+}
+
+// the default form:
+//  - long instruction
+//  - 1 to 3 sources in slots 0, 1, 2
+//  - address & flags
+void
+CodeEmitterNV50::emitForm_MAD(const Instruction *i)
+{
+   assert(i->encSize == 8);
+   code[0] |= 1;
+
+   emitFlagsRd(i);
+   emitFlagsWr(i);
+
+   setDst(i, 0);
+
+   if (i->srcExists(0))
+      emitSrc0(i->src[0]);
+
+   if (i->srcExists(1))
+      emitSrc1(i->src[1]);
+
+   if (i->srcExists(2))
+      emitSrc2(i->src[2]);
+
+   setAReg16(i, 1);
+}
+
+// like default form, but 2nd source in slot 2, and no 3rd source
+void
+CodeEmitterNV50::emitForm_ADD(const Instruction *i)
+{
+   assert(i->encSize == 8);
+   code[0] |= 1;
+
+   emitFlagsRd(i);
+   emitFlagsWr(i);
+
+   setDst(i, 0);
+
+   if (i->srcExists(0))
+      emitSrc0(i->src[0]);
+
+   if (i->srcExists(1))
+      emitSrc2(i->src[1]);
+
+   setAReg16(i, 1);
+}
+
+// default short form
+void
+CodeEmitterNV50::emitForm_MUL(const Instruction *i)
+{
+   assert(i->encSize == 4 && !(code[0] & 1));
+   assert(i->defExists(0));
+   assert(!i->getPredicate());
+
+   setDst(i, 0);
+
+   if (i->srcExists(0))
+      emitSrc0(i->src[0]);
+
+   if (i->srcExists(1))
+      emitSrc1(i->src[1]);
+}
+
+// usual immediate form
+// - 1 to 3 sources where last is immediate
+// - no address or predicate possible
+void
+CodeEmitterNV50::emitForm_IMM(const Instruction *i)
+{
+   assert(i->encSize == 8);
+   code[0] |= 1;
+
+   assert(i->defExists(0) && i->srcExists(0));
+
+   setDst(i, 0);
+
+   if (i->srcExists(2)) {
+      emitSrc0(i->src[0]);
+      emitSrc1(i->src[1]);
+      setImmediate(i, 2);
+   } else
+   if (i->srcExists(1)) {
+      emitSrc0(i->src[0]);
+      setImmediate(i, 1);
+   } else {
+      setImmediate(i, 0);
+   }
+}
+
+void
+CodeEmitterNV50::emitLoadStoreSize(DataType ty, int pos)
+{
+   uint8_t enc;
+
+   switch (ty) {
+   case TYPE_F32: // fall through
+   case TYPE_S32: // fall through
+   case TYPE_U32:  enc = 0x6; break;
+   case TYPE_B128: enc = 0x5; break;
+   case TYPE_F64:  enc = 0x4; break;
+   case TYPE_S16:  enc = 0x3; break;
+   case TYPE_U16:  enc = 0x2; break;
+   case TYPE_S8:   enc = 0x1; break;
+   case TYPE_U8:   enc = 0x0; break;
+   default:
+      enc = 0;
+      assert(!"invalid load/store type");
+      break;
+   }
+   code[pos / 32] |= enc << (pos % 32);
+}
+
+void
+CodeEmitterNV50::emitLOAD(const Instruction *i)
+{
+   DataFile sf = i->src[0].getFile();
+
+   switch (sf) {
+   case FILE_SHADER_INPUT:
+      code[0] = 0x10000001;
+      code[1] = 0x04200000 | (i->lanes << 14);
+      break;
+   case FILE_MEMORY_CONST:
+      code[0] = 0x10000001;
+      code[1] = 0x24000000 | (i->getSrc(0)->reg.fileIndex << 22);
+      break;
+   case FILE_MEMORY_LOCAL:
+      code[0] = 0xd0000001;
+      code[1] = 0x40000000;
+      break;
+   case FILE_MEMORY_GLOBAL:
+      code[0] = 0xd0000001 | (i->getSrc(0)->reg.fileIndex << 16);
+      code[1] = 0x80000000;
+      break;
+   default:
+      assert(!"invalid load source file");
+      break;
+   }
+   if (sf == FILE_MEMORY_LOCAL ||
+       sf == FILE_MEMORY_GLOBAL)
+      emitLoadStoreSize(i->sType, 21 + 32);
+
+   setDst(i, 0);
+
+   emitFlagsRd(i);
+   emitFlagsWr(i);
+
+   if (i->src[0].getFile() == FILE_MEMORY_GLOBAL) {
+      srcId(*i->src[0].getIndirect(0), 9);
+   } else {
+      setAReg16(i, 0);
+      srcAddr16(i->src[0], 9);
+   }
+}
+
+void
+CodeEmitterNV50::emitSTORE(const Instruction *i)
+{
+   DataFile f = i->getSrc(0)->reg.file;
+   int32_t offset = i->getSrc(0)->reg.data.offset;
+
+   switch (f) {
+   case FILE_SHADER_OUTPUT:
+      code[0] = 0x00000001 | ((offset >> 2) << 2);
+      code[1] = 0x80c00000;
+      srcId(i->src[1], 32 + 15);
+      break;
+   case FILE_MEMORY_GLOBAL:
+      code[0] = 0xd0000000;
+      code[1] = 0xa0000000;
+      emitLoadStoreSize(i->dType, 21 + 32);
+      break;
+   case FILE_MEMORY_LOCAL:
+      code[0] = 0xd0000001;
+      code[1] = 0x60000000;
+      emitLoadStoreSize(i->dType, 21 + 32);
+      break;
+   case FILE_MEMORY_SHARED:
+      code[0] = 0x00000001;
+      code[1] = 0xe0000000;
+      switch (typeSizeof(i->dType)) {
+      case 1:
+         code[0] |= offset << 9;
+         code[1] |= 0x00400000;
+         break;
+      case 2:
+         code[0] |= (offset >> 1) << 9;
+         break;
+      case 4:
+         code[0] |= (offset >> 2) << 9;
+         code[1] |= 0x04000000;
+         break;
+      default:
+         assert(0);
+         break;
+      }
+      break;
+   default:
+      assert(!"invalid store destination file");
+      break;
+   }
+
+   if (f != FILE_SHADER_OUTPUT) {
+      srcId(i->src[1], 2);
+      if (f == FILE_MEMORY_GLOBAL)
+         srcId(*i->src[0].getIndirect(0), 9);
+      if (f == FILE_MEMORY_LOCAL)
+         srcAddr16(i->src[0], 9);
+   }
+   if (f != FILE_MEMORY_GLOBAL)
+      setAReg16(i, 0);
+
+   emitFlagsRd(i);
+}
+
+void
+CodeEmitterNV50::emitMOV(const Instruction *i)
+{
+   DataFile sf = i->getSrc(0)->reg.file;
+   DataFile df = i->getDef(0)->reg.file;
+
+   assert(sf == FILE_GPR || df == FILE_GPR);
+
+   if (sf == FILE_FLAGS) {
+      code[0] = 0x00000001;
+      code[1] = 0x20000000;
+      defId(i->def[0], 2);
+      srcId(i->src[0], 12);
+      emitFlagsRd(i);
+   } else
+   if (sf == FILE_ADDRESS) {
+      code[0] = 0x00000001;
+      code[1] = 0x40000000;
+      defId(i->def[0], 2);
+      setARegBits(SDATA(i->src[0]).id + 1);
+   } else
+   if (df == FILE_FLAGS) {
+      code[0] = 0x00000001;
+      code[1] = 0xa0000000;
+      defId(i->def[0], 4);
+      srcId(i->src[0], 9);
+      emitFlagsRd(i);
+   } else
+   if (sf == FILE_IMMEDIATE) {
+      code[0] = 0x10008001;
+      code[1] = 0x00000003;
+      emitForm_IMM(i);
+   } else {
+      if (i->encSize == 4) {
+         code[0] = 0x10008000;
+      } else {
+         code[0] = 0x10000001;
+         code[1] = 0x04000000 | (i->lanes << 14);
+      }
+      defId(i->def[0], 2);
+      srcId(i->src[0], 9);
+   }
+   if (df == FILE_SHADER_OUTPUT) {
+      assert(i->encSize == 8);
+      code[1] |= 0x8;
+   }
+}
+
+void
+CodeEmitterNV50::emitNOP()
+{
+   code[0] = 0xf0000001;
+   code[1] = 0xe0000000;
+}
+
+void
+CodeEmitterNV50::emitQUADOP(const Instruction *i, uint8_t lane, uint8_t quOp)
+{
+   code[0] = 0xc0000000 | (lane << 16);
+   code[1] = 0x80000000;
+
+   code[0] |= (quOp & 0x03) << 20;
+   code[1] |= (quOp & 0xfc) << 20;
+
+   emitForm_ADD(i);
+
+   if (!i->srcExists(1))
+      srcId(i->src[0], 32 + 14);
+}
+
+void
+CodeEmitterNV50::emitPFETCH(const Instruction *i)
+{
+   code[0] = 0x11800001;
+   code[1] = 0x04200000 | (0xf << 14);
+
+   defId(i->def[0], 2);
+   srcAddr8(i->src[0], 9);
+   setAReg16(i, 0);
+}
+
+void
+CodeEmitterNV50::emitINTERP(const Instruction *i)
+{
+   code[0] = 0x80000000;
+
+   defId(i->def[0], 2);
+   srcAddr8(i->src[0], 16);
+
+   if (i->getInterpMode() == NV50_IR_INTERP_FLAT) {
+      code[0] |= 1 << 8;
+   } else {
+      if (i->op == OP_PINTERP) {
+         code[0] |= 1 << 25;
+         srcId(i->src[1], 9);
+      }
+      if (i->getSampleMode() == NV50_IR_INTERP_CENTROID)
+         code[0] |= 1 << 24;
+   }
+
+   if (i->encSize == 8) {
+      emitFlagsRd(i);
+      code[1] |=
+         (code[0] & (3 << 24)) >> (24 - 16) |
+         (code[0] & (1 <<  8)) >> (18 -  8);
+      code[0] &= ~0x03000100;
+      code[0] |= 1;
+   }
+}
+
+void
+CodeEmitterNV50::emitMINMAX(const Instruction *i)
+{
+   if (i->dType == TYPE_F64) {
+      code[0] = 0xe0000000;
+      code[1] = (i->op == OP_MIN) ? 0xa0000000 : 0xc0000000;
+   } else {
+      code[0] = 0x30000000;
+      code[1] = 0x80000000;
+      if (i->op == OP_MIN)
+         code[1] |= 0x20000000;
+
+      switch (i->dType) {
+      case TYPE_F32: code[0] |= 0x80000000; break;
+      case TYPE_S32: code[1] |= 0x8c000000; break;
+      case TYPE_U32: code[1] |= 0x84000000; break;
+      case TYPE_S16: code[1] |= 0x80000000; break;
+      case TYPE_U16: break;
+      default:
+         assert(0);
+         break;
+      }
+      code[1] |= i->src[0].mod.abs() << 20;
+      code[1] |= i->src[1].mod.abs() << 19;
+   }
+   emitForm_MAD(i);
+}
+
+void
+CodeEmitterNV50::emitFMAD(const Instruction *i)
+{
+   const int neg_mul = i->src[0].mod.neg() ^ i->src[1].mod.neg();
+   const int neg_add = i->src[2].mod.neg();
+
+   code[0] = 0xe0000000;
+
+   if (i->encSize == 4) {
+      emitForm_MUL(i);
+      assert(!neg_mul && !neg_add);
+   } else {
+      emitForm_MAD(i);
+      code[1] |= neg_mul << 26;
+      code[1] |= neg_add << 27;
+      if (i->saturate)
+         code[1] |= 1 << 29;
+   }
+}
+
+void
+CodeEmitterNV50::emitFADD(const Instruction *i)
+{
+   const int neg0 = i->src[0].mod.neg();
+   const int neg1 = i->src[1].mod.neg() ^ ((i->op == OP_SUB) ? 1 : 0);
+
+   code[0] = 0xb0000000;
+
+   assert(!(i->src[0].mod | i->src[1].mod).abs());
+
+   if (i->src[1].getFile() == FILE_IMMEDIATE) {
+      emitForm_IMM(i);
+      code[0] |= neg0 << 15;
+      code[0] |= neg1 << 22;
+   } else
+   if (i->encSize == 8) {
+      emitForm_ADD(i);
+      code[1] |= neg0 << 26;
+      code[1] |= neg1 << 27;
+      if (i->saturate)
+         code[1] |= 1 << 29;
+   } else {
+      emitForm_MUL(i);
+      code[0] |= neg0 << 15;
+      code[0] |= neg1 << 22;
+   }
+}
+
+void
+CodeEmitterNV50::emitUADD(const Instruction *i)
+{
+   code[0] = 0x20008000;
+
+   if (i->src[0].getFile() == FILE_IMMEDIATE) {
+      emitForm_IMM(i);
+   } else
+   if (i->encSize == 8) {
+      code[0] = 0x20000000;
+      code[1] = 0x04000000;
+      emitForm_ADD(i);
+   } else {
+      emitForm_MUL(i);
+   }
+   assert(!(i->src[0].mod.neg() && i->src[1].mod.neg()));
+   code[0] |= i->src[0].mod.neg() << 28;
+   code[0] |= i->src[1].mod.neg() << 22;
+}
+
+void
+CodeEmitterNV50::emitAADD(const Instruction *i)
+{
+   const int s = (i->op == OP_MOV) ? 0 : 1;
+
+   code[0] = 0xd0000001 | (i->getSrc(s)->reg.data.u16 << 9);
+   code[1] = 0x20000000;
+
+   code[0] |= (DDATA(i->def[0]).id + 1) << 2;
+
+   emitFlagsRd(i);
+
+   if (s && i->srcExists(0))
+      setARegBits(SDATA(i->src[0]).id + 1);
+}
+
+void
+CodeEmitterNV50::emitFMUL(const Instruction *i)
+{
+   const int neg = (i->src[0].mod ^ i->src[1].mod).neg();
+
+   code[0] = 0xc0000000;
+
+   if (i->src[0].getFile() == FILE_IMMEDIATE) {
+      emitForm_IMM(i);
+      if (neg)
+         code[0] |= 0x8000;
+   } else
+   if (i->encSize == 8) {
+      emitForm_MAD(i);
+      if (neg)
+         code[1] |= 0x08000000;
+   } else {
+      emitForm_MUL(i);
+      if (neg)
+         code[0] |= 0x8000;
+   }
+}
+
+void
+CodeEmitterNV50::emitSET(const Instruction *i)
+{
+   code[0] = 0x30000000;
+   code[1] = 0x60000000;
+
+   emitCondCode(i->asCmp()->setCond, 32 + 14);
+
+   switch (i->sType) {
+   case TYPE_F32: code[0] |= 0x80000000; break;
+   case TYPE_S32: code[1] |= 0x0c000000; break;
+   case TYPE_U32: code[1] |= 0x04000000; break;
+   case TYPE_S16: code[1] |= 0x08000000; break;
+   case TYPE_U16: break;
+   default:
+      assert(0);
+      break;
+   }
+   emitForm_MAD(i);
+}
+
+void
+CodeEmitterNV50::roundMode_CVT(RoundMode rnd)
+{
+   switch (rnd) {
+   case ROUND_NI: code[1] |= 0x08000000; break;
+   case ROUND_M:  code[1] |= 0x00020000; break;
+   case ROUND_MI: code[1] |= 0x08020000; break;
+   case ROUND_P:  code[1] |= 0x00040000; break;
+   case ROUND_PI: code[1] |= 0x08040000; break;
+   case ROUND_Z:  code[1] |= 0x00060000; break;
+   case ROUND_ZI: code[1] |= 0x08060000; break;
+   default:
+      assert(rnd == ROUND_N);
+      break;
+   }
+}
+
+void
+CodeEmitterNV50::emitCVT(const Instruction *i)
+{
+   const bool f2f = isFloatType(i->dType) && isFloatType(i->sType);
+   RoundMode rnd;
+
+   switch (i->op) {
+   case OP_CEIL:  rnd = f2f ? ROUND_PI : ROUND_P; break;
+   case OP_FLOOR: rnd = f2f ? ROUND_MI : ROUND_M; break;
+   case OP_TRUNC: rnd = f2f ? ROUND_ZI : ROUND_Z; break;
+   default:
+      rnd = i->rnd;
+      break;
+   }
+
+   code[0] = 0xa0000000;
+
+   switch (i->dType) {
+   case TYPE_F64:
+      switch (i->sType) {
+      case TYPE_F64: code[1] = 0xc4404000; break;
+      case TYPE_S64: code[1] = 0x44414000; break;
+      case TYPE_U64: code[1] = 0x44404000; break;
+      case TYPE_F32: code[1] = 0xc4400000; break;
+      case TYPE_S32: code[1] = 0x44410000; break;
+      case TYPE_U32: code[1] = 0x44400000; break;
+      default:
+         assert(0);
+         break;
+      }
+      break;
+   case TYPE_S64:
+      switch (i->sType) {
+      case TYPE_F64: code[1] = 0x8c404000; break;
+      case TYPE_F32: code[1] = 0x8c400000; break;
+      default:
+         assert(0);
+         break;
+      }
+      break;
+   case TYPE_U64:
+      switch (i->sType) {
+      case TYPE_F64: code[1] = 0x84404000; break;
+      case TYPE_F32: code[1] = 0x84400000; break;
+      default:
+         assert(0);
+         break;
+      }
+      break;
+   case TYPE_F32:
+      switch (i->sType) {
+      case TYPE_F64: code[1] = 0xc0404000; break;
+      case TYPE_S64: code[1] = 0x40414000; break;
+      case TYPE_U64: code[1] = 0x40404000; break;
+      case TYPE_F32: code[1] = 0xc4004000; break;
+      case TYPE_S32: code[1] = 0x44014000; break;
+      case TYPE_U32: code[1] = 0x44004000; break;
+      case TYPE_F16: code[1] = 0xc4000000; break;
+      default:
+         assert(0);
+         break;
+      }
+      break;
+   case TYPE_S32:
+      switch (i->sType) {
+      case TYPE_F64: code[1] = 0x88404000; break;
+      case TYPE_F32: code[1] = 0x8c004000; break;
+      case TYPE_S32: code[1] = 0x0c014000; break;
+      case TYPE_U32: code[1] = 0x0c004000; break;
+      case TYPE_F16: code[1] = 0x8c000000; break;
+      case TYPE_S16: code[1] = 0x0c010000; break;
+      case TYPE_U16: code[1] = 0x0c000000; break;
+      case TYPE_S8:  code[1] = 0x0c018000; break;
+      case TYPE_U8:  code[1] = 0x0c008000; break;
+      default:
+         assert(0);
+         break;
+      }
+      break;
+   case TYPE_U32:
+      switch (i->sType) {
+      case TYPE_F64: code[1] = 0x80404000; break;
+      case TYPE_F32: code[1] = 0x84004000; break;
+      case TYPE_S32: code[1] = 0x04014000; break;
+      case TYPE_U32: code[1] = 0x04004000; break;
+      case TYPE_F16: code[1] = 0x84000000; break;
+      case TYPE_S16: code[1] = 0x04010000; break;
+      case TYPE_U16: code[1] = 0x04000000; break;
+      case TYPE_S8:  code[1] = 0x04018000; break;
+      case TYPE_U8:  code[1] = 0x04008000; break;
+      default:
+         assert(0);
+         break;
+      }
+   case TYPE_S16:
+   case TYPE_U16:
+   case TYPE_S8:
+   case TYPE_U8:
+   default:
+      assert(0);
+      break;
+   }
+   if (typeSizeof(i->sType) == 1 && i->getSrc(0)->reg.size == 4)
+      code[1] |= 0x00004000;
+
+   roundMode_CVT(rnd);
+
+   switch (i->op) {
+   case OP_ABS: code[1] |= 1 << 20; break;
+   case OP_SAT: code[1] |= 1 << 19; break;
+   case OP_NEG: code[1] |= 1 << 29; break;
+   default:
+      break;
+   }
+   code[1] ^= i->src[0].mod.neg() << 29;
+   code[1] |= i->src[0].mod.abs() << 20;
+   if (i->saturate)
+      code[1] |= 1 << 19;
+
+   assert(i->op != OP_ABS || !i->src[0].mod.neg());
+
+   emitForm_MAD(i);
+}
+
+void
+CodeEmitterNV50::emitPreOp(const Instruction *i)
+{
+   code[0] = 0xb0000000;
+   code[1] = (i->op == OP_PREEX2) ? 0xc0004000 : 0xc0000000;
+
+   code[1] |= i->src[0].mod.abs() << 20;
+   code[1] |= i->src[0].mod.neg() << 26;
+
+   emitForm_MAD(i);
+}
+
+void
+CodeEmitterNV50::emitSFnOp(const Instruction *i, uint8_t subOp)
+{
+   code[0] = 0x90000000;
+
+   if (i->encSize == 4) {
+      assert(i->op == OP_RCP);
+      emitForm_MUL(i);
+   } else {
+      code[1] = subOp << 29;
+      code[1] |= i->src[0].mod.abs() << 20;
+      code[1] |= i->src[0].mod.neg() << 26;
+      emitForm_MAD(i);
+   }
+}
+
+void
+CodeEmitterNV50::emitLogicOp(const Instruction *i)
+{
+   code[0] = 0xd0000000;
+
+   if (i->src[1].getFile() == FILE_IMMEDIATE) {
+      switch (i->op) {
+      case OP_OR:  code[0] |= 0x0100; break;
+      case OP_XOR: code[0] |= 0x8000; break;
+      default:
+         assert(i->op == OP_AND);
+         break;
+      }
+      emitForm_IMM(i);
+   } else {
+      switch (i->op) {
+      case OP_AND: code[1] = 0x04000000; break;
+      case OP_OR:  code[1] = 0x04004000; break;
+      case OP_XOR: code[1] = 0x04008000; break;
+      default:
+         assert(0);
+         break;
+      }
+      emitForm_MAD(i);
+   }
+}
+
+void
+CodeEmitterNV50::emitARL(const Instruction *i)
+{
+   assert(i->src[1].getFile() == FILE_IMMEDIATE);
+
+   code[0] = 0x00000001 | (i->getSrc(1)->reg.data.u32 & 0x3f) << 16;
+   code[1] = 0xc0000000;
+
+   code[0] |= (DDATA(i->def[0]).id + 1) << 2;
+   emitSrc0(i->src[0]);
+   emitFlagsRd(i);
+}
+
+void
+CodeEmitterNV50::emitShift(const Instruction *i)
+{
+   if (i->def[0].getFile() == FILE_ADDRESS) {
+      emitARL(i);
+   } else {
+      code[0] = 0x30000001;
+      code[1] = (i->op == OP_SHR) ? 0xe4000000 : 0xc4000000;
+      if (isSignedType(i->sType))
+          code[1] |= 1 << 27;
+
+      if (i->src[1].getFile() == FILE_IMMEDIATE) {
+         code[1] |= 1 << 20;
+         code[0] |= (i->getSrc(1)->reg.data.u32 & 0x7f) << 16;
+         emitFlagsRd(i);
+      } else {
+         emitForm_MAD(i);
+      }
+   }
+}
+
+void
+CodeEmitterNV50::emitOUT(const Instruction *i)
+{
+   code[0] = (i->op == OP_EMIT) ? 0xf0000200 : 0xf0000400;
+   code[1] = 0xc0000001;
+
+   emitFlagsRd(i);
+}
+
+void
+CodeEmitterNV50::emitTEX(const TexInstruction *i)
+{
+   code[0] = 0xf0000001;
+   code[1] = 0x00000000;
+
+   switch (i->op) {
+   case OP_TXB:
+      code[1] = 0x20000000;
+      break;
+   case OP_TXL:
+      code[1] = 0x40000000;
+      break;
+   case OP_TXF:
+      code[0] = 0x01000000;
+      break;
+   case OP_TXG:
+      code[0] = 0x01000000;
+      code[1] = 0x80000000;
+      break;
+   default:
+      assert(i->op == OP_TEX);
+      break;
+   }
+
+   code[0] |= i->tex.r << 9;
+   code[0] |= i->tex.s << 17;
+
+   int argc = i->tex.target.getArgCount();
+
+   if (i->op == OP_TXB || i->op == OP_TXL)
+      argc += 1;
+   if (i->tex.target.isShadow())
+      argc += 1;
+   assert(argc <= 4);
+
+   code[0] |= (argc - 1) << 22;
+
+   if (i->tex.target.isCube()) {
+      code[0] |= 0x08000000;
+   } else
+   if (i->tex.useOffsets) {
+      code[1] |= (i->tex.offset[0][0] & 0xf) << 16;
+      code[1] |= (i->tex.offset[0][1] & 0xf) << 20;
+      code[1] |= (i->tex.offset[0][2] & 0xf) << 24;
+   }
+
+   code[0] |= (i->tex.mask & 0x3) << 25;
+   code[1] |= (i->tex.mask & 0xc) << 12;
+
+   if (i->tex.liveOnly)
+      code[1] |= 4;
+
+   defId(i->def[0], 2);
+
+   emitFlagsRd(i);
+}
+
+void
+CodeEmitterNV50::emitFlow(const Instruction *i, uint8_t flowOp)
+{
+   const FlowInstruction *f = i->asFlow();
+
+   code[0] = 0x00000003 | (flowOp << 28);
+   code[1] = 0x00000000;
+
+   emitFlagsRd(i);
+
+   if (f && f->target.bb) {
+      uint32_t pos;
+
+      if (f->op == OP_CALL) {
+         if (f->builtin) {
+            pos = 0; // XXX: TODO
+         } else {
+            pos = f->target.fn->binPos;
+         }
+      } else {
+         pos = f->target.bb->binPos;
+      }
+
+      code[0] |= ((pos >>  2) & 0xffff) << 11;
+      code[1] |= ((pos >> 18) & 0x003f) << 14;
+   }
+}
+
+bool
+CodeEmitterNV50::emitInstruction(Instruction *insn)
+{
+   if (!insn->encSize) {
+      ERROR("skipping unencodable instruction: "); insn->print();
+      return false;
+   } else
+   if (codeSize + insn->encSize > codeSizeLimit) {
+      ERROR("code emitter output buffer too small\n");
+      return false;
+   }
+
+   switch (insn->op) {
+   case OP_MOV:
+      emitMOV(insn);
+      break;
+   case OP_NOP:
+   case OP_JOIN:
+      emitNOP();
+      break;
+   case OP_VFETCH:
+   case OP_LOAD:
+      emitLOAD(insn);
+      break;
+   case OP_EXPORT:
+   case OP_STORE:
+      emitSTORE(insn);
+      break;
+   case OP_PFETCH:
+      emitPFETCH(insn);
+      break;
+   case OP_LINTERP:
+   case OP_PINTERP:
+      emitINTERP(insn);
+      break;
+   case OP_ADD:
+   case OP_SUB:
+      if (isFloatType(insn->dType))
+         emitFADD(insn);
+      else
+         emitUADD(insn);
+      break;
+   case OP_MUL:
+      if (isFloatType(insn->dType))
+         emitFMUL(insn);
+      else
+         emitUMUL(insn);
+      break;
+   case OP_MAD:
+   case OP_FMA:
+      emitFMAD(insn);
+      break;
+      break;
+   case OP_AND:
+   case OP_OR:
+   case OP_XOR:
+      emitLogicOp(insn);
+      break;
+   case OP_MIN:
+   case OP_MAX:
+      emitMINMAX(insn);
+      break;
+   case OP_CEIL:
+   case OP_FLOOR:
+   case OP_TRUNC:
+   case OP_CVT:
+      emitCVT(insn);
+      break;
+   case OP_RCP:
+      emitSFnOp(insn, 0);
+      break;
+   case OP_RSQ:
+      emitSFnOp(insn, 2);
+      break;
+   case OP_LG2:
+      emitSFnOp(insn, 3);
+      break;
+   case OP_SIN:
+      emitSFnOp(insn, 4);
+      break;
+   case OP_COS:
+      emitSFnOp(insn, 5);
+      break;
+   case OP_EX2:
+      emitSFnOp(insn, 6);
+      break;
+   case OP_PRESIN:
+   case OP_PREEX2:
+      emitPreOp(insn);
+      break;
+   case OP_TEX:
+   case OP_TXB:
+   case OP_TXL:
+      emitTEX(insn->asTex());
+      break;
+   case OP_EMIT:
+   case OP_RESTART:
+      emitOUT(insn);
+      break;
+   case OP_DISCARD:
+      emitFlow(insn, 0x0);
+      break;
+   case OP_BRA:
+      emitFlow(insn, 0x1);
+      break;
+   case OP_CALL:
+      emitFlow(insn, 0x2);
+      break;
+   case OP_RET:
+      emitFlow(insn, 0x3);
+      break;
+   case OP_PREBREAK:
+      emitFlow(insn, 0x4);
+      break;
+   case OP_BREAK:
+      emitFlow(insn, 0x5);
+      break;
+   case OP_QUADON:
+      emitFlow(insn, 0x6);
+      break;
+   case OP_QUADPOP:
+      emitFlow(insn, 0x7);
+      break;
+   case OP_JOINAT:
+      emitFlow(insn, 0xa);
+      break;
+   case OP_PRERET:
+      emitFlow(insn, 0xd);
+      break;
+   case OP_QUADOP:
+      emitQUADOP(insn, insn->lanes, insn->subOp);
+      break;
+   case OP_DFDX:
+      emitQUADOP(insn, 4, insn->src[0].mod.neg() ? 0x66 : 0x99);
+      break;
+   case OP_DFDY:
+      emitQUADOP(insn, 5, insn->src[0].mod.neg() ? 0x5a : 0xa5);
+      break;
+   case OP_PHI:
+   case OP_UNION:
+   case OP_CONSTRAINT:
+      ERROR("operation should have been eliminated");
+      return false;
+   case OP_EXP:
+   case OP_LOG:
+   case OP_SQRT:
+   case OP_POW:
+   case OP_SELP:
+   case OP_SLCT:
+   case OP_TXD:
+   case OP_PRECONT:
+   case OP_CONT:
+   case OP_POPCNT:
+   case OP_INSBF:
+   case OP_EXTBF:
+      ERROR("operation should have been lowered\n");
+      return false;
+   default:
+      ERROR("unknow op\n");
+      return false;
+   }
+   if (insn->join)
+      code[1] |= 0x2;
+   else
+   if (insn->exit)
+      code[1] |= 0x1;
+
+   assert((insn->encSize == 8) == (code[1] & 1));
+
+   code += insn->encSize / 4;
+   codeSize += insn->encSize;
+   return true;
+}
+
+uint32_t
+CodeEmitterNV50::getMinEncodingSize(const Instruction *i) const
+{
+   const Target::OpInfo &info = targ->getOpInfo(i);
+
+   if (info.minEncSize == 8)
+      return 8;
+
+   return 4;
+}
+
+CodeEmitterNV50::CodeEmitterNV50(const Target *target) : targ(target)
+{
+   code = NULL;
+   codeSize = codeSizeLimit = 0;
+}
+
+CodeEmitter *
+Target::getCodeEmitter(Program::Type type)
+{
+   CodeEmitterNV50 *emit = new CodeEmitterNV50(this);
+   emit->setProgramType(type);
+   return emit;
+}
+
+} // namespace nv50_ir
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_from_tgsi.cpp b/src/gallium/drivers/nv50/codegen/nv50_ir_from_tgsi.cpp
new file mode 100644
index 00000000000..c2f464de31b
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_from_tgsi.cpp
@@ -0,0 +1,2288 @@
+
+extern "C" {
+#include "tgsi/tgsi_dump.h"
+#include "tgsi/tgsi_scan.h"
+}
+
+#include "nv50_ir.h"
+#include "nv50_ir_util.h"
+#include "nv50_ir_build_util.h"
+
+namespace tgsi {
+
+class Source;
+
+static nv50_ir::operation translateOpcode(uint opcode);
+static nv50_ir::DataFile translateFile(uint file);
+static nv50_ir::TexTarget translateTexture(uint texTarg);
+static nv50_ir::SVSemantic translateSysVal(uint sysval);
+
+class Instruction
+{
+public:
+   Instruction(const struct tgsi_full_instruction *inst) : insn(inst) { }
+
+   class SrcRegister
+   {
+   public:
+      SrcRegister(const struct tgsi_full_src_register *src)
+         : reg(src->Register),
+           fsr(src)
+      { }
+
+      SrcRegister(const struct tgsi_src_register& src) : reg(src), fsr(NULL) { }
+
+      struct tgsi_src_register offsetToSrc(struct tgsi_texture_offset off)
+      {
+         struct tgsi_src_register reg;
+         memset(&reg, 0, sizeof(reg));
+         reg.Index = off.Index;
+         reg.File = off.File;
+         reg.SwizzleX = off.SwizzleX;
+         reg.SwizzleY = off.SwizzleY;
+         reg.SwizzleZ = off.SwizzleZ;
+         return reg;
+      }
+
+      SrcRegister(const struct tgsi_texture_offset& off) :
+         reg(offsetToSrc(off)),
+         fsr(NULL)
+      { }
+
+      uint getFile() const { return reg.File; }
+
+      bool is2D() const { return reg.Dimension; }
+
+      bool isIndirect(int dim) const
+      {
+         return (dim && fsr) ? fsr->Dimension.Indirect : reg.Indirect;
+      }
+
+      int getIndex(int dim) const
+      {
+         return (dim && fsr) ? fsr->Dimension.Index : reg.Index;
+      }
+
+      int getSwizzle(int chan) const
+      {
+         return tgsi_util_get_src_register_swizzle(&reg, chan);
+      }
+
+      nv50_ir::Modifier getMod(int chan) const;
+
+      SrcRegister getIndirect(int dim) const
+      {
+         assert(fsr && isIndirect(dim));
+         if (dim)
+            return SrcRegister(fsr->DimIndirect);
+         return SrcRegister(fsr->Indirect);
+      }
+
+      uint32_t getValueU32(int c, const struct nv50_ir_prog_info *info) const
+      {
+         assert(reg.File == TGSI_FILE_IMMEDIATE);
+         assert(!reg.Absolute);
+         assert(!reg.Negate);
+         return info->immd.data[reg.Index * 4 + getSwizzle(c)];
+      }
+
+   private:
+      const struct tgsi_src_register reg;
+      const struct tgsi_full_src_register *fsr;
+   };
+
+   class DstRegister
+   {
+   public:
+      DstRegister(const struct tgsi_full_dst_register *dst)
+         : reg(dst->Register),
+           fdr(dst)
+      { }
+
+      DstRegister(const struct tgsi_dst_register& dst) : reg(dst), fdr(NULL) { }
+
+      uint getFile() const { return reg.File; }
+
+      bool is2D() const { return reg.Dimension; }
+
+      bool isIndirect(int dim) const
+      {
+         return (dim && fdr) ? fdr->Dimension.Indirect : reg.Indirect;
+      }
+
+      int getIndex(int dim) const
+      {
+         return (dim && fdr) ? fdr->Dimension.Dimension : reg.Index;
+      }
+
+      unsigned int getMask() const { return reg.WriteMask; }
+
+      bool isMasked(int chan) const { return !(getMask() & (1 << chan)); }
+
+      SrcRegister getIndirect(int dim) const
+      {
+         assert(fdr && isIndirect(dim));
+         if (dim)
+            return SrcRegister(fdr->DimIndirect);
+         return SrcRegister(fdr->Indirect);
+      }
+
+   private:
+      const struct tgsi_dst_register reg;
+      const struct tgsi_full_dst_register *fdr;
+   };
+
+   inline uint getOpcode() const { return insn->Instruction.Opcode; }
+
+   unsigned int srcCount() const { return insn->Instruction.NumSrcRegs; }
+   unsigned int dstCount() const { return insn->Instruction.NumDstRegs; }
+
+   // mask of used components of source s
+   unsigned int srcMask(unsigned int s) const;
+
+   SrcRegister getSrc(unsigned int s) const
+   {
+      assert(s < srcCount());
+      return SrcRegister(&insn->Src[s]);
+   }
+
+   DstRegister getDst(unsigned int d) const
+   {
+      assert(d < dstCount());
+      return DstRegister(&insn->Dst[d]);
+   }
+
+   SrcRegister getTexOffset(unsigned int i) const
+   {
+      assert(i < TGSI_FULL_MAX_TEX_OFFSETS);
+      return SrcRegister(insn->TexOffsets[i]);
+   }
+
+   unsigned int getNumTexOffsets() const { return insn->Texture.NumOffsets; }
+
+   bool checkDstSrcAliasing() const;
+
+   inline nv50_ir::operation getOP() const {
+      return translateOpcode(getOpcode()); }
+
+   nv50_ir::DataType inferSrcType() const;
+   nv50_ir::DataType inferDstType() const;
+
+   nv50_ir::CondCode getSetCond() const;
+
+   nv50_ir::TexInstruction::Target getTexture(const Source *, int s) const;
+
+   inline uint getLabel() { return insn->Label.Label; }
+
+   unsigned getSaturate() const { return insn->Instruction.Saturate; }
+
+   void print() const
+   {
+      tgsi_dump_instruction(insn, 1);
+   }
+
+private:
+   const struct tgsi_full_instruction *insn;
+};
+
+unsigned int Instruction::srcMask(unsigned int s) const
+{
+   unsigned int mask = insn->Dst[0].Register.WriteMask;
+
+   switch (insn->Instruction.Opcode) {
+   case TGSI_OPCODE_COS:
+   case TGSI_OPCODE_SIN:
+      return (mask & 0x8) | ((mask & 0x7) ? 0x1 : 0x0);
+   case TGSI_OPCODE_DP3:
+      return 0x7;
+   case TGSI_OPCODE_DP4:
+   case TGSI_OPCODE_DPH:
+   case TGSI_OPCODE_KIL: /* WriteMask ignored */
+      return 0xf;
+   case TGSI_OPCODE_DST:
+      return mask & (s ? 0xa : 0x6);
+   case TGSI_OPCODE_EX2:
+   case TGSI_OPCODE_EXP:
+   case TGSI_OPCODE_LG2:
+   case TGSI_OPCODE_LOG:
+   case TGSI_OPCODE_POW:
+   case TGSI_OPCODE_RCP:
+   case TGSI_OPCODE_RSQ:
+   case TGSI_OPCODE_SCS:
+      return 0x1;
+   case TGSI_OPCODE_IF:
+      return 0x1;
+   case TGSI_OPCODE_LIT:
+      return 0xb;
+   case TGSI_OPCODE_TEX:
+   case TGSI_OPCODE_TXB:
+   case TGSI_OPCODE_TXD:
+   case TGSI_OPCODE_TXL:
+   case TGSI_OPCODE_TXP:
+   {
+      const struct tgsi_instruction_texture *tex = &insn->Texture;
+
+      assert(insn->Instruction.Texture);
+
+      mask = 0x7;
+      if (insn->Instruction.Opcode != TGSI_OPCODE_TEX &&
+          insn->Instruction.Opcode != TGSI_OPCODE_TXD)
+         mask |= 0x8; /* bias, lod or proj */
+
+      switch (tex->Texture) {
+      case TGSI_TEXTURE_1D:
+         mask &= 0x9;
+         break;
+      case TGSI_TEXTURE_SHADOW1D:
+         mask &= 0x5;
+         break;
+      case TGSI_TEXTURE_1D_ARRAY:
+      case TGSI_TEXTURE_2D:
+      case TGSI_TEXTURE_RECT:
+         mask &= 0xb;
+         break;
+      default:
+         break;
+      }
+   }
+      return mask;
+   case TGSI_OPCODE_XPD:
+   {
+      unsigned int x = 0;
+      if (mask & 1) x |= 0x6;
+      if (mask & 2) x |= 0x5;
+      if (mask & 4) x |= 0x3;
+      return x;
+   }
+   default:
+      break;
+   }
+
+   return mask;
+}
+
+nv50_ir::Modifier Instruction::SrcRegister::getMod(int chan) const
+{
+   nv50_ir::Modifier m(0);
+
+   if (reg.Absolute)
+      m = m | nv50_ir::Modifier(NV50_IR_MOD_ABS);
+   if (reg.Negate)
+      m = m | nv50_ir::Modifier(NV50_IR_MOD_NEG);
+   return m;
+}
+
+static nv50_ir::DataFile translateFile(uint file)
+{
+   switch (file) {
+   case TGSI_FILE_CONSTANT:        return nv50_ir::FILE_MEMORY_CONST;
+   case TGSI_FILE_INPUT:           return nv50_ir::FILE_SHADER_INPUT;
+   case TGSI_FILE_OUTPUT:          return nv50_ir::FILE_SHADER_OUTPUT;
+   case TGSI_FILE_TEMPORARY:       return nv50_ir::FILE_GPR;
+   case TGSI_FILE_ADDRESS:         return nv50_ir::FILE_ADDRESS;
+   case TGSI_FILE_PREDICATE:       return nv50_ir::FILE_PREDICATE;
+   case TGSI_FILE_IMMEDIATE:       return nv50_ir::FILE_IMMEDIATE;
+   case TGSI_FILE_SYSTEM_VALUE:    return nv50_ir::FILE_SYSTEM_VALUE;
+   case TGSI_FILE_IMMEDIATE_ARRAY: return nv50_ir::FILE_IMMEDIATE;
+   case TGSI_FILE_TEMPORARY_ARRAY: return nv50_ir::FILE_MEMORY_LOCAL;
+   case TGSI_FILE_RESOURCE:        return nv50_ir::FILE_MEMORY_GLOBAL;
+   case TGSI_FILE_SAMPLER:
+   case TGSI_FILE_NULL:
+   default:
+      return nv50_ir::FILE_NULL;
+   }
+}
+
+static nv50_ir::SVSemantic translateSysVal(uint sysval)
+{
+   switch (sysval) {
+   case TGSI_SEMANTIC_FACE:       return nv50_ir::SV_FACE;
+   case TGSI_SEMANTIC_PSIZE:      return nv50_ir::SV_POINT_SIZE;
+   case TGSI_SEMANTIC_PRIMID:     return nv50_ir::SV_PRIMITIVE_ID;
+   case TGSI_SEMANTIC_INSTANCEID: return nv50_ir::SV_INSTANCE_ID;
+   default:
+      assert(0);
+      return nv50_ir::SV_CLOCK;
+   }
+}
+
+#define NV50_IR_TEX_TARG_CASE(a, b) \
+   case TGSI_TEXTURE_##a: return nv50_ir::TEX_TARGET_##b;
+
+static nv50_ir::TexTarget translateTexture(uint tex)
+{
+   switch (tex) {
+   NV50_IR_TEX_TARG_CASE(1D, 1D);
+   NV50_IR_TEX_TARG_CASE(2D, 2D);
+   NV50_IR_TEX_TARG_CASE(3D, 3D);
+   NV50_IR_TEX_TARG_CASE(CUBE, CUBE);
+   NV50_IR_TEX_TARG_CASE(RECT, RECT);
+   NV50_IR_TEX_TARG_CASE(1D_ARRAY, 1D_ARRAY);
+   NV50_IR_TEX_TARG_CASE(2D_ARRAY, 2D_ARRAY);
+   NV50_IR_TEX_TARG_CASE(SHADOW1D, 1D_SHADOW);
+   NV50_IR_TEX_TARG_CASE(SHADOW2D, 2D_SHADOW);
+   NV50_IR_TEX_TARG_CASE(SHADOW1D_ARRAY, 1D_ARRAY_SHADOW);
+   NV50_IR_TEX_TARG_CASE(SHADOW2D_ARRAY, 2D_ARRAY_SHADOW);
+   NV50_IR_TEX_TARG_CASE(SHADOWRECT, RECT_SHADOW);
+
+   case TGSI_TEXTURE_UNKNOWN:
+   default:
+      assert(!"invalid texture target");
+      return nv50_ir::TEX_TARGET_2D;
+   }
+}
+   
+nv50_ir::DataType Instruction::inferSrcType() const
+{
+   switch (getOpcode()) {
+   case TGSI_OPCODE_AND:
+   case TGSI_OPCODE_OR:
+   case TGSI_OPCODE_XOR:
+   case TGSI_OPCODE_U2F:
+   case TGSI_OPCODE_UADD:
+   case TGSI_OPCODE_UDIV:
+   case TGSI_OPCODE_UMOD:
+   case TGSI_OPCODE_UMAD:
+   case TGSI_OPCODE_UMUL:
+   case TGSI_OPCODE_UMAX:
+   case TGSI_OPCODE_UMIN:
+   case TGSI_OPCODE_USEQ:
+   case TGSI_OPCODE_USGE:
+   case TGSI_OPCODE_USLT:
+   case TGSI_OPCODE_USNE:
+   case TGSI_OPCODE_USHR:
+   case TGSI_OPCODE_UCMP:
+      return nv50_ir::TYPE_U32;
+   case TGSI_OPCODE_I2F:
+   case TGSI_OPCODE_IDIV:
+   case TGSI_OPCODE_IMAX:
+   case TGSI_OPCODE_IMIN:
+   case TGSI_OPCODE_INEG:
+   case TGSI_OPCODE_ISGE:
+   case TGSI_OPCODE_ISHR:
+   case TGSI_OPCODE_ISLT:
+   case TGSI_OPCODE_SAD: // not sure about SAD, but no one has a float version
+   case TGSI_OPCODE_MOD:
+   case TGSI_OPCODE_UARL:
+      return nv50_ir::TYPE_S32;
+   default:
+      return nv50_ir::TYPE_F32;
+   }
+}
+
+nv50_ir::DataType Instruction::inferDstType() const
+{
+   switch (getOpcode()) {
+   case TGSI_OPCODE_F2U: return nv50_ir::TYPE_U32;
+   case TGSI_OPCODE_F2I: return nv50_ir::TYPE_S32;
+   case TGSI_OPCODE_I2F:
+   case TGSI_OPCODE_U2F:
+      return nv50_ir::TYPE_F32;
+   default:
+      return inferSrcType();
+   }
+}
+
+nv50_ir::CondCode Instruction::getSetCond() const
+{
+   using namespace nv50_ir;
+
+   switch (getOpcode()) {
+   case TGSI_OPCODE_SLT:
+   case TGSI_OPCODE_ISLT:
+   case TGSI_OPCODE_USLT:
+      return CC_LT;
+   case TGSI_OPCODE_SLE:
+      return CC_LE;
+   case TGSI_OPCODE_SGE:
+   case TGSI_OPCODE_ISGE:
+   case TGSI_OPCODE_USGE:
+      return CC_GE;
+   case TGSI_OPCODE_SGT:
+      return CC_GT;
+   case TGSI_OPCODE_SEQ:
+   case TGSI_OPCODE_USEQ:
+      return CC_EQ;
+   case TGSI_OPCODE_SNE:
+   case TGSI_OPCODE_USNE:
+      return CC_NE;
+   case TGSI_OPCODE_SFL:
+      return CC_NEVER;
+   case TGSI_OPCODE_STR:
+   default:
+      return CC_ALWAYS;
+   }
+}
+
+#define NV50_IR_OPCODE_CASE(a, b) case TGSI_OPCODE_##a: return nv50_ir::OP_##b
+   
+static nv50_ir::operation translateOpcode(uint opcode)
+{
+   switch (opcode) {
+   NV50_IR_OPCODE_CASE(ARL, SHL);
+   NV50_IR_OPCODE_CASE(MOV, MOV);
+
+   NV50_IR_OPCODE_CASE(RCP, RCP);
+   NV50_IR_OPCODE_CASE(RSQ, RSQ);
+
+   NV50_IR_OPCODE_CASE(MUL, MUL);
+   NV50_IR_OPCODE_CASE(ADD, ADD);
+
+   NV50_IR_OPCODE_CASE(MIN, MIN);
+   NV50_IR_OPCODE_CASE(MAX, MAX);
+   NV50_IR_OPCODE_CASE(SLT, SET);
+   NV50_IR_OPCODE_CASE(SGE, SET);
+   NV50_IR_OPCODE_CASE(MAD, MAD);
+   NV50_IR_OPCODE_CASE(SUB, SUB);
+
+   NV50_IR_OPCODE_CASE(FLR, FLOOR);
+   NV50_IR_OPCODE_CASE(ROUND, CVT);
+   NV50_IR_OPCODE_CASE(EX2, EX2);
+   NV50_IR_OPCODE_CASE(LG2, LG2);
+   NV50_IR_OPCODE_CASE(POW, POW);
+
+   NV50_IR_OPCODE_CASE(ABS, ABS);
+
+   NV50_IR_OPCODE_CASE(COS, COS);
+   NV50_IR_OPCODE_CASE(DDX, DFDX);
+   NV50_IR_OPCODE_CASE(DDY, DFDY);
+   NV50_IR_OPCODE_CASE(KILP, DISCARD);
+
+   NV50_IR_OPCODE_CASE(SEQ, SET);
+   NV50_IR_OPCODE_CASE(SFL, SET);
+   NV50_IR_OPCODE_CASE(SGT, SET);
+   NV50_IR_OPCODE_CASE(SIN, SIN);
+   NV50_IR_OPCODE_CASE(SLE, SET);
+   NV50_IR_OPCODE_CASE(SNE, SET);
+   NV50_IR_OPCODE_CASE(STR, SET);
+   NV50_IR_OPCODE_CASE(TEX, TEX);
+   NV50_IR_OPCODE_CASE(TXD, TXD);
+   NV50_IR_OPCODE_CASE(TXP, TEX);
+
+   NV50_IR_OPCODE_CASE(BRA, BRA);
+   NV50_IR_OPCODE_CASE(CAL, CALL);
+   NV50_IR_OPCODE_CASE(RET, RET);
+   NV50_IR_OPCODE_CASE(CMP, SLCT);
+
+   NV50_IR_OPCODE_CASE(TXB, TXB);
+
+   NV50_IR_OPCODE_CASE(DIV, DIV);
+
+   NV50_IR_OPCODE_CASE(TXL, TXL);
+
+   NV50_IR_OPCODE_CASE(CEIL, CEIL);
+   NV50_IR_OPCODE_CASE(I2F, CVT);
+   NV50_IR_OPCODE_CASE(NOT, NOT);
+   NV50_IR_OPCODE_CASE(TRUNC, TRUNC);
+   NV50_IR_OPCODE_CASE(SHL, SHL);
+
+   NV50_IR_OPCODE_CASE(AND, AND);
+   NV50_IR_OPCODE_CASE(OR, OR);
+   NV50_IR_OPCODE_CASE(MOD, MOD);
+   NV50_IR_OPCODE_CASE(XOR, XOR);
+   NV50_IR_OPCODE_CASE(SAD, SAD);
+   NV50_IR_OPCODE_CASE(TXF, TXF);
+   NV50_IR_OPCODE_CASE(TXQ, TXQ);
+
+   NV50_IR_OPCODE_CASE(EMIT, EMIT);
+   NV50_IR_OPCODE_CASE(ENDPRIM, RESTART);
+
+   NV50_IR_OPCODE_CASE(KIL, DISCARD);
+
+   NV50_IR_OPCODE_CASE(F2I, CVT);
+   NV50_IR_OPCODE_CASE(IDIV, DIV);
+   NV50_IR_OPCODE_CASE(IMAX, MAX);
+   NV50_IR_OPCODE_CASE(IMIN, MIN);
+   NV50_IR_OPCODE_CASE(INEG, NEG);
+   NV50_IR_OPCODE_CASE(ISGE, SET);
+   NV50_IR_OPCODE_CASE(ISHR, SHR);
+   NV50_IR_OPCODE_CASE(ISLT, SET);
+   NV50_IR_OPCODE_CASE(F2U, CVT);
+   NV50_IR_OPCODE_CASE(U2F, CVT);
+   NV50_IR_OPCODE_CASE(UADD, ADD);
+   NV50_IR_OPCODE_CASE(UDIV, DIV);
+   NV50_IR_OPCODE_CASE(UMAD, MAD);
+   NV50_IR_OPCODE_CASE(UMAX, MAX);
+   NV50_IR_OPCODE_CASE(UMIN, MIN);
+   NV50_IR_OPCODE_CASE(UMOD, MOD);
+   NV50_IR_OPCODE_CASE(UMUL, MUL);
+   NV50_IR_OPCODE_CASE(USEQ, SET);
+   NV50_IR_OPCODE_CASE(USGE, SET);
+   NV50_IR_OPCODE_CASE(USHR, SHR);
+   NV50_IR_OPCODE_CASE(USLT, SET);
+   NV50_IR_OPCODE_CASE(USNE, SET);
+
+   NV50_IR_OPCODE_CASE(LOAD, TXF);
+   NV50_IR_OPCODE_CASE(SAMPLE, TEX);
+   NV50_IR_OPCODE_CASE(SAMPLE_B, TXB);
+   NV50_IR_OPCODE_CASE(SAMPLE_C, TEX);
+   NV50_IR_OPCODE_CASE(SAMPLE_C_LZ, TEX);
+   NV50_IR_OPCODE_CASE(SAMPLE_D, TXD);
+   NV50_IR_OPCODE_CASE(SAMPLE_L, TXL);
+   NV50_IR_OPCODE_CASE(GATHER4, TXG);
+   NV50_IR_OPCODE_CASE(RESINFO, TXQ);
+
+   NV50_IR_OPCODE_CASE(END, EXIT);
+
+   default:
+      return nv50_ir::OP_NOP;
+   }
+}
+
+bool Instruction::checkDstSrcAliasing() const
+{
+   if (insn->Dst[0].Register.Indirect) // no danger if indirect, using memory
+      return false;
+
+   for (int s = 0; s < TGSI_FULL_MAX_SRC_REGISTERS; ++s) {
+      if (insn->Src[s].Register.File == TGSI_FILE_NULL)
+         break;
+      if (insn->Src[s].Register.File == insn->Dst[0].Register.File &&
+          insn->Src[s].Register.Index == insn->Dst[0].Register.Index)
+         return true;
+   }
+   return false;
+}
+
+class Source
+{
+public:
+   Source(struct nv50_ir_prog_info *);
+   ~Source();
+
+   struct Subroutine
+   {
+      unsigned pc;
+   };
+
+public:
+   bool scanSource();
+   unsigned fileSize(unsigned file) const { return scan.file_max[file] + 1; }
+
+public:
+   struct tgsi_shader_info scan;
+   struct tgsi_full_instruction *insns;
+   const struct tgsi_token *tokens;
+   struct nv50_ir_prog_info *info;
+
+   nv50_ir::DynArray tempArrays;
+   nv50_ir::DynArray immdArrays;
+   int tempArrayCount;
+   int immdArrayCount;
+
+   bool mainTempsInLMem;
+
+   uint8_t *resourceTargets; // TGSI_TEXTURE_*
+   unsigned resourceCount;
+
+   Subroutine *subroutines;
+   unsigned subroutineCount;
+
+private:
+   int inferSysValDirection(unsigned sn) const;
+   bool scanDeclaration(const struct tgsi_full_declaration *);
+   bool scanInstruction(const struct tgsi_full_instruction *);
+   void scanProperty(const struct tgsi_full_property *);
+   void scanImmediate(const struct tgsi_full_immediate *);
+
+   inline bool isEdgeFlagPassthrough(const Instruction&) const;
+};
+
+Source::Source(struct nv50_ir_prog_info *prog) : info(prog)
+{
+   tokens = (const struct tgsi_token *)info->bin.source;
+
+   if (prog->dbgFlags & NV50_IR_DEBUG_BASIC)
+      tgsi_dump(tokens, 0);
+
+   resourceTargets = NULL;
+   subroutines = NULL;
+
+   mainTempsInLMem = FALSE;
+}
+
+Source::~Source()
+{
+   if (insns)
+      FREE(insns);
+
+   if (info->immd.data)
+      FREE(info->immd.data);
+   if (info->immd.type)
+      FREE(info->immd.type);
+
+   if (resourceTargets)
+      delete[] resourceTargets;
+   if (subroutines)
+      delete[] subroutines;
+}
+
+bool Source::scanSource()
+{
+   unsigned insnCount = 0;
+   unsigned subrCount = 0;
+   struct tgsi_parse_context parse;
+
+   tgsi_scan_shader(tokens, &scan);
+
+   insns = (struct tgsi_full_instruction *)MALLOC(scan.num_instructions *
+                                                  sizeof(insns[0]));
+   if (!insns)
+      return false;
+
+   resourceCount = scan.file_max[TGSI_FILE_RESOURCE] + 1;
+   resourceTargets = new uint8_t[resourceCount];
+
+   subroutineCount = scan.opcode_count[TGSI_OPCODE_BGNSUB] + 1;
+   subroutines = new Subroutine[subroutineCount];
+
+   info->immd.bufSize = 0;
+   tempArrayCount = 0;
+   immdArrayCount = 0;
+
+   info->numInputs = scan.file_max[TGSI_FILE_INPUT] + 1;
+   info->numOutputs = scan.file_max[TGSI_FILE_OUTPUT] + 1;
+   info->numSysVals = scan.file_max[TGSI_FILE_SYSTEM_VALUE] + 1;
+
+   if (info->type == PIPE_SHADER_FRAGMENT) {
+      info->prop.fp.writesDepth = scan.writes_z;
+      info->prop.fp.usesDiscard = scan.uses_kill;
+   } else
+   if (info->type == PIPE_SHADER_GEOMETRY) {
+      info->prop.gp.instanceCount = 1; // default value
+   }
+
+   info->immd.data = (uint32_t *)MALLOC(scan.immediate_count * 16);
+   info->immd.type = (ubyte *)MALLOC(scan.immediate_count * sizeof(ubyte));
+
+   tgsi_parse_init(&parse, tokens);
+   while (!tgsi_parse_end_of_tokens(&parse)) {
+      tgsi_parse_token(&parse);
+
+      switch (parse.FullToken.Token.Type) {
+      case TGSI_TOKEN_TYPE_IMMEDIATE:
+         scanImmediate(&parse.FullToken.FullImmediate);
+         break;
+      case TGSI_TOKEN_TYPE_DECLARATION:
+         scanDeclaration(&parse.FullToken.FullDeclaration);
+         break;
+      case TGSI_TOKEN_TYPE_INSTRUCTION:
+         insns[insnCount++] = parse.FullToken.FullInstruction;
+         if (insns[insnCount - 1].Instruction.Opcode == TGSI_OPCODE_BGNSUB)
+            subroutines[++subrCount].pc = insnCount - 1;
+         else
+            scanInstruction(&parse.FullToken.FullInstruction);
+         break;
+      case TGSI_TOKEN_TYPE_PROPERTY:
+         scanProperty(&parse.FullToken.FullProperty);
+         break;
+      default:
+         INFO("unknown TGSI token type: %d\n", parse.FullToken.Token.Type);
+         break;
+      }
+   }
+   tgsi_parse_free(&parse);
+
+   if (mainTempsInLMem)
+      info->bin.tlsSpace += (scan.file_max[TGSI_FILE_TEMPORARY] + 1) * 16;
+
+   return info->assignSlots(info) == 0;
+}
+
+void Source::scanProperty(const struct tgsi_full_property *prop)
+{
+   switch (prop->Property.PropertyName) {
+   case TGSI_PROPERTY_GS_OUTPUT_PRIM:
+      info->prop.gp.outputPrim = prop->u[0].Data;
+      break;
+   case TGSI_PROPERTY_GS_INPUT_PRIM:
+      info->prop.gp.inputPrim = prop->u[0].Data;
+      break;
+   case TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES:
+      info->prop.gp.maxVertices = prop->u[0].Data;
+      break;
+#if 0
+   case TGSI_PROPERTY_GS_INSTANCE_COUNT:
+      info->prop.gp.instanceCount = prop->u[0].Data;
+      break;
+#endif
+   case TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS:
+      info->prop.fp.separateFragData = TRUE;
+      break;
+   case TGSI_PROPERTY_FS_COORD_ORIGIN:
+   case TGSI_PROPERTY_FS_COORD_PIXEL_CENTER:
+      // we don't care
+      break;
+   default:
+      INFO("unhandled TGSI property %d\n", prop->Property.PropertyName);
+      break;
+   }
+}
+
+void Source::scanImmediate(const struct tgsi_full_immediate *imm)
+{
+   const unsigned n = info->immd.count++;
+
+   assert(n < scan.immediate_count);
+
+   for (int c = 0; c < 4; ++c)
+      info->immd.data[n * 4 + c] = imm->u[c].Uint;
+
+   info->immd.type[n] = imm->Immediate.DataType;
+}
+
+int Source::inferSysValDirection(unsigned sn) const
+{
+   switch (sn) {
+   case TGSI_SEMANTIC_INSTANCEID:
+// case TGSI_SEMANTIC_VERTEXID:
+      return 1;
+#if 0
+   case TGSI_SEMANTIC_LAYER:
+   case TGSI_SEMANTIC_VIEWPORTINDEX:
+      return 0;
+#endif
+   case TGSI_SEMANTIC_PRIMID:
+      return (info->type == PIPE_SHADER_FRAGMENT) ? 1 : 0;
+   default:
+      return 0;
+   }
+}
+
+bool Source::scanDeclaration(const struct tgsi_full_declaration *decl)
+{
+   unsigned i;
+   unsigned sn = TGSI_SEMANTIC_GENERIC;
+   unsigned si = 0;
+   const unsigned first = decl->Range.First, last = decl->Range.Last;
+
+   if (decl->Declaration.Semantic) {
+      sn = decl->Semantic.Name;
+      si = decl->Semantic.Index;
+   }
+
+   switch (decl->Declaration.File) {
+   case TGSI_FILE_INPUT:
+      if (info->type == PIPE_SHADER_VERTEX) {
+         // all vertex attributes are equal
+         for (i = first; i <= last; ++i) {
+            info->in[i].sn = TGSI_SEMANTIC_GENERIC;
+            info->in[i].si = i;
+         }
+      } else {
+         for (i = first; i <= last; ++i, ++si) {
+            info->in[i].id = i;
+            info->in[i].sn = sn;
+            info->in[i].si = si;
+            if (info->type == PIPE_SHADER_FRAGMENT) {
+               // translate interpolation mode
+               switch (decl->Declaration.Interpolate) {
+               case TGSI_INTERPOLATE_CONSTANT:
+                  info->in[i].flat = 1;
+                  break;
+               case TGSI_INTERPOLATE_LINEAR:
+                  if (sn != TGSI_SEMANTIC_COLOR) // GL_NICEST
+                     info->in[i].linear = 1;
+                  break;
+               default:
+                  break;
+               }
+               if (decl->Declaration.Centroid)
+                  info->in[i].centroid = 1;
+            }
+         }
+      }
+      break;
+   case TGSI_FILE_OUTPUT:
+      for (i = first; i <= last; ++i, ++si) {
+         switch (sn) {
+         case TGSI_SEMANTIC_POSITION:
+            if (info->type == PIPE_SHADER_FRAGMENT)
+               info->io.fragDepth = i;
+            break;
+         case TGSI_SEMANTIC_COLOR:
+            if (info->type == PIPE_SHADER_FRAGMENT)
+               info->prop.fp.numColourResults++;
+            break;
+         case TGSI_SEMANTIC_EDGEFLAG:
+            info->io.edgeFlagOut = i;
+            break;
+         default:
+            break;
+         }
+         info->out[i].id = i;
+         info->out[i].sn = sn;
+         info->out[i].si = si;
+      }
+      break;
+   case TGSI_FILE_SYSTEM_VALUE:
+      for (i = first; i <= last; ++i, ++si) {
+         info->sv[i].sn = sn;
+         info->sv[i].si = si;
+         info->sv[i].input = inferSysValDirection(sn);
+      }
+      break;
+   case TGSI_FILE_RESOURCE:
+      for (i = first; i <= last; ++i)
+         resourceTargets[i] = decl->Resource.Resource;
+      break;
+   case TGSI_FILE_IMMEDIATE_ARRAY:
+   {
+      if (decl->Dim.Index2D >= immdArrayCount)
+         immdArrayCount = decl->Dim.Index2D + 1;
+      immdArrays[decl->Dim.Index2D].u32 = (last + 1) << 2;
+      int c;
+      uint32_t base, count;
+      switch (decl->Declaration.UsageMask) {
+      case 0x1: c = 1; break;
+      case 0x3: c = 2; break;
+      default:
+         c = 4;
+         break;
+      }
+      immdArrays[decl->Dim.Index2D].u32 |= c;
+      count = (last + 1) * c;
+      base = info->immd.bufSize / 4;
+      info->immd.bufSize = (info->immd.bufSize + count * 4 + 0xf) & ~0xf;
+      info->immd.buf = (uint32_t *)REALLOC(info->immd.buf, base * 4,
+                                           info->immd.bufSize);
+      // NOTE: this assumes array declarations are ordered by Dim.Index2D
+      for (i = 0; i < count; ++i)
+         info->immd.buf[base + i] = decl->ImmediateData.u[i].Uint;
+   }
+      break;
+   case TGSI_FILE_TEMPORARY_ARRAY:
+   {
+      if (decl->Dim.Index2D >= tempArrayCount)
+         tempArrayCount = decl->Dim.Index2D + 1;
+      tempArrays[decl->Dim.Index2D].u32 = (last + 1) << 2;
+      int c;
+      uint32_t count;
+      switch (decl->Declaration.UsageMask) {
+      case 0x1: c = 1; break;
+      case 0x3: c = 2; break;
+      default:
+         c = 4;
+         break;
+      }
+      tempArrays[decl->Dim.Index2D].u32 |= c;
+      count = (last + 1) * c;
+      info->bin.tlsSpace += (info->bin.tlsSpace + count * 4 + 0xf) & ~0xf;
+   }
+      break;
+   case TGSI_FILE_NULL:
+   case TGSI_FILE_TEMPORARY:
+   case TGSI_FILE_ADDRESS:
+   case TGSI_FILE_CONSTANT:
+   case TGSI_FILE_IMMEDIATE:
+   case TGSI_FILE_PREDICATE:
+   case TGSI_FILE_SAMPLER:
+      break;
+   default:
+      ERROR("unhandled TGSI_FILE %d\n", decl->Declaration.File);
+      return false;
+   }
+   return true;
+}
+
+inline bool Source::isEdgeFlagPassthrough(const Instruction& insn) const
+{
+   return insn.getOpcode() == TGSI_OPCODE_MOV &&
+      insn.getDst(0).getIndex(0) == info->io.edgeFlagOut &&
+      insn.getSrc(0).getFile() == TGSI_FILE_INPUT;
+}
+
+bool Source::scanInstruction(const struct tgsi_full_instruction *inst)
+{
+   Instruction insn(inst);
+
+   if (insn.dstCount()) {
+      if (insn.getDst(0).getFile() == TGSI_FILE_OUTPUT) {
+         Instruction::DstRegister dst = insn.getDst(0);
+
+         if (dst.isIndirect(0))
+            for (unsigned i = 0; i < info->numOutputs; ++i)
+               info->out[i].mask = 0xf;
+         else
+            info->out[dst.getIndex(0)].mask |= dst.getMask();
+
+         if (isEdgeFlagPassthrough(insn))
+            info->io.edgeFlagIn = insn.getSrc(0).getIndex(0);
+      } else
+      if (insn.getDst(0).getFile() == TGSI_FILE_TEMPORARY) {
+         if (insn.getDst(0).isIndirect(0))
+            mainTempsInLMem = TRUE;
+      }
+   }
+
+   for (unsigned s = 0; s < insn.srcCount(); ++s) {
+      Instruction::SrcRegister src = insn.getSrc(s);
+      if (src.getFile() == TGSI_FILE_TEMPORARY)
+         if (src.isIndirect(0))
+            mainTempsInLMem = TRUE;
+      if (src.getFile() != TGSI_FILE_INPUT)
+         continue;
+      unsigned mask = insn.srcMask(s);
+
+      if (src.isIndirect(0)) {
+         for (unsigned i = 0; i < info->numInputs; ++i)
+            info->in[i].mask = 0xf;
+      } else {
+         for (unsigned c = 0; c < 4; ++c) {
+            if (!(mask & (1 << c)))
+               continue;
+            int k = src.getSwizzle(c);
+            int i = src.getIndex(0);
+            if (info->in[i].sn != TGSI_SEMANTIC_FOG || k == TGSI_SWIZZLE_X)
+               if (k <= TGSI_SWIZZLE_W)
+                  info->in[i].mask |= 1 << k;
+         }
+      }
+   }
+   return true;
+}
+
+nv50_ir::TexInstruction::Target
+Instruction::getTexture(const tgsi::Source *code, int s) const
+{
+   if (insn->Instruction.Texture) {
+      return translateTexture(insn->Texture.Texture);
+   } else {
+      // XXX: indirect access
+      unsigned int r = getSrc(s).getIndex(0);
+      assert(r < code->resourceCount);
+      return translateTexture(code->resourceTargets[r]);
+   }
+}
+
+} // namespace tgsi
+
+namespace {
+
+using namespace nv50_ir;
+
+class Converter : public BuildUtil
+{
+public:
+   Converter(Program *, const tgsi::Source *);
+   ~Converter();
+
+   bool run();
+
+private:
+   Value *getVertexBase(int s);
+   Value *fetchSrc(int s, int c);
+   Value *acquireDst(int d, int c);
+   void storeDst(int d, int c, Value *);
+
+   Value *fetchSrc(const tgsi::Instruction::SrcRegister src, int c, Value *ptr);
+   void storeDst(const tgsi::Instruction::DstRegister dst, int c,
+                 Value *val, Value *ptr);
+
+   Value *applySrcMod(Value *, int s, int c);
+
+   Symbol *makeSym(uint file, int fileIndex, int idx, int c, uint32_t addr);
+   Symbol *srcToSym(tgsi::Instruction::SrcRegister, int c);
+   Symbol *dstToSym(tgsi::Instruction::DstRegister, int c);
+
+   bool handleInstruction(const struct tgsi_full_instruction *);
+   void exportOutputs();
+   inline bool isEndOfSubroutine(uint ip);
+
+   void loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask);
+
+   // R,S,L,C,Dx,Dy encode TGSI sources for respective values (0xSf for auto)
+   void setTexRS(TexInstruction *, unsigned int& s, int R, int S);
+   void handleTEX(Value *dst0[4], int R, int S, int L, int C, int Dx, int Dy);
+   void handleTXF(Value *dst0[4], int R);
+   void handleTXQ(Value *dst0[4], enum TexQuery);
+   void handleLIT(Value *dst0[4]);
+   void handleUserClipPlanes();
+
+   Value *interpolate(tgsi::Instruction::SrcRegister, int c, Value *ptr);
+
+   void insertConvergenceOps(BasicBlock *conv, BasicBlock *fork);
+
+   Value *buildDot(int dim);
+
+private:
+   const struct tgsi::Source *code;
+   const struct nv50_ir_prog_info *info;
+
+   uint ip; // instruction pointer
+
+   tgsi::Instruction tgsi;
+
+   DataType dstTy;
+   DataType srcTy;
+
+   DataArray tData; // TGSI_FILE_TEMPORARY
+   DataArray aData; // TGSI_FILE_ADDRESS
+   DataArray pData; // TGSI_FILE_PREDICATE
+   DataArray oData; // TGSI_FILE_OUTPUT (if outputs in registers)
+   DataArray *lData; // TGSI_FILE_TEMPORARY_ARRAY
+   DataArray *iData; // TGSI_FILE_IMMEDIATE_ARRAY
+
+   Value *zero;
+   Value *fragCoord[4];
+   Value *clipVtx[4];
+
+   Value *vtxBase[5]; // base address of vertex in primitive (for TP/GP)
+   uint8_t vtxBaseValid;
+
+   Stack condBBs;  // fork BB, then else clause BB
+   Stack joinBBs;  // fork BB, for inserting join ops on ENDIF
+   Stack loopBBs;  // loop headers
+   Stack breakBBs; // end of / after loop
+   Stack entryBBs; // start of current (inlined) subroutine
+   Stack leaveBBs; // end of current (inlined) subroutine
+   Stack retIPs;   // return instruction pointer
+};
+
+Symbol *
+Converter::srcToSym(tgsi::Instruction::SrcRegister src, int c)
+{
+   const int swz = src.getSwizzle(c);
+
+   return makeSym(src.getFile(),
+                  src.is2D() ? src.getIndex(1) : 0,
+                  src.isIndirect(0) ? -1 : src.getIndex(0), swz,
+                  src.getIndex(0) * 16 + swz * 4);
+}
+
+Symbol *
+Converter::dstToSym(tgsi::Instruction::DstRegister dst, int c)
+{
+   return makeSym(dst.getFile(),
+                  dst.is2D() ? dst.getIndex(1) : 0,
+                  dst.isIndirect(0) ? -1 : dst.getIndex(0), c,
+                  dst.getIndex(0) * 16 + c * 4);
+}
+
+Symbol *
+Converter::makeSym(uint tgsiFile, int fileIdx, int idx, int c, uint32_t address)
+{
+   Symbol *sym = new_Symbol(prog, tgsi::translateFile(tgsiFile));
+
+   sym->reg.fileIndex = fileIdx;
+
+   if (idx >= 0) {
+      if (sym->reg.file == FILE_SHADER_INPUT)
+         sym->setOffset(info->in[idx].slot[c] * 4);
+      else
+      if (sym->reg.file == FILE_SHADER_OUTPUT)
+         sym->setOffset(info->out[idx].slot[c] * 4);
+      else
+      if (sym->reg.file == FILE_SYSTEM_VALUE)
+         sym->setSV(tgsi::translateSysVal(info->sv[idx].sn), c);
+      else
+         sym->setOffset(address);
+   } else {
+      sym->setOffset(address);
+   }
+   return sym;
+}
+
+static inline uint8_t
+translateInterpMode(const struct nv50_ir_varying *var, operation& op)
+{
+   uint8_t mode;
+
+   if (var->flat)
+      mode = NV50_IR_INTERP_FLAT;
+   else
+   if (var->linear)
+      mode = NV50_IR_INTERP_LINEAR;
+   else
+      mode = NV50_IR_INTERP_PERSPECTIVE;
+
+   op = (mode == NV50_IR_INTERP_PERSPECTIVE) ? OP_PINTERP : OP_LINTERP;
+
+   if (var->centroid)
+      mode |= NV50_IR_INTERP_CENTROID;
+
+   return mode;
+}
+
+Value *
+Converter::interpolate(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
+{
+   operation op;
+
+   // XXX: no way to know interpolation mode if we don't know what's accessed
+   const uint8_t mode = translateInterpMode(&info->in[ptr ? 0 :
+                                                      src.getIndex(0)], op);
+
+   Instruction *insn = new_Instruction(func, op, TYPE_F32);
+
+   insn->setDef(0, getScratch());
+   insn->setSrc(0, srcToSym(src, c));
+   if (op == OP_PINTERP)
+      insn->setSrc(1, fragCoord[3]);
+   if (ptr)
+      insn->setIndirect(0, 0, ptr);
+
+   insn->setInterpolate(mode);
+
+   bb->insertTail(insn);
+   return insn->getDef(0);
+}
+
+Value *
+Converter::applySrcMod(Value *val, int s, int c)
+{
+   Modifier m = tgsi.getSrc(s).getMod(c);
+   DataType ty = tgsi.inferSrcType();
+
+   if (m & Modifier(NV50_IR_MOD_ABS))
+      val = mkOp1v(OP_ABS, ty, getScratch(), val);
+
+   if (m & Modifier(NV50_IR_MOD_NEG))
+      val = mkOp1v(OP_NEG, ty, getScratch(), val);
+
+   return val;
+}
+
+Value *
+Converter::getVertexBase(int s)
+{
+   assert(s < 5);
+   if (!(vtxBaseValid & (1 << s))) {
+      const int index = tgsi.getSrc(s).getIndex(1);
+      Value *rel = NULL;
+      if (tgsi.getSrc(s).isIndirect(1))
+         rel = fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL);
+      vtxBaseValid |= 1 << s;
+      vtxBase[s] = mkOp2v(OP_PFETCH, TYPE_U32, getSSA(), mkImm(index), rel);
+   }
+   return vtxBase[s];
+}
+
+Value *
+Converter::fetchSrc(int s, int c)
+{
+   Value *res;
+   Value *ptr = NULL, *dimRel = NULL;
+
+   tgsi::Instruction::SrcRegister src = tgsi.getSrc(s);
+
+   if (src.isIndirect(0))
+      ptr = fetchSrc(src.getIndirect(0), 0, NULL);
+
+   if (src.is2D()) {
+      switch (src.getFile()) {
+      case TGSI_FILE_INPUT:
+         dimRel = getVertexBase(s);
+         break;
+      case TGSI_FILE_CONSTANT:
+         // on NVC0, this is valid and c{I+J}[k] == cI[(J << 16) + k]
+         if (src.isIndirect(1))
+            dimRel = fetchSrc(src.getIndirect(1), 0, 0);
+         break;
+      default:
+         break;
+      }
+   }
+
+   res = fetchSrc(src, c, ptr);
+
+   if (dimRel)
+      res->getInsn()->setIndirect(0, 1, dimRel);
+
+   return applySrcMod(res, s, c);
+}
+
+Value *
+Converter::fetchSrc(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
+{
+   const int idx = src.getIndex(0);
+   const int swz = src.getSwizzle(c);
+
+   switch (src.getFile()) {
+   case TGSI_FILE_TEMPORARY:
+      return tData.load(idx, swz, ptr);
+   case TGSI_FILE_PREDICATE:
+      return pData.load(idx, swz, ptr);
+   case TGSI_FILE_ADDRESS:
+      return aData.load(idx, swz, ptr);
+
+   case TGSI_FILE_TEMPORARY_ARRAY:
+      assert(src.is2D() && src.getIndex(1) < code->tempArrayCount);
+      return lData[src.getIndex(1)].load(idx, swz, ptr);
+   case TGSI_FILE_IMMEDIATE_ARRAY:
+      assert(src.is2D() && src.getIndex(1) < code->immdArrayCount);
+      return iData[src.getIndex(1)].load(idx, swz, ptr);
+
+   case TGSI_FILE_IMMEDIATE:
+      assert(!ptr);
+      return loadImm(NULL, info->immd.data[idx * 4 + swz]);
+
+   case TGSI_FILE_CONSTANT:
+      return mkLoad(TYPE_U32, srcToSym(src, c), ptr);
+
+   case TGSI_FILE_INPUT:
+      if (prog->getType() == Program::TYPE_FRAGMENT) {
+         // don't load masked inputs, won't be assigned a slot
+         if (!ptr && !(info->in[idx].mask & (1 << swz)))
+            return loadImm(NULL, swz == TGSI_SWIZZLE_W ? 1.0f : 0.0f);
+         return interpolate(src, c, ptr);
+      }
+      return mkLoad(TYPE_U32, srcToSym(src, c), ptr);
+
+   case TGSI_FILE_SYSTEM_VALUE:
+      assert(!ptr);
+      return mkOp1v(OP_RDSV, TYPE_U32, getSSA(), srcToSym(src, c));
+
+   case TGSI_FILE_OUTPUT:
+   case TGSI_FILE_RESOURCE:
+   case TGSI_FILE_SAMPLER:
+   case TGSI_FILE_NULL:
+   default:
+      assert(!"invalid/unhandled TGSI source file");
+      return NULL;
+   }
+}
+
+Value *
+Converter::acquireDst(int d, int c)
+{
+   const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
+
+   if (dst.isMasked(c))
+      return NULL;
+   if (dst.isIndirect(0))
+      return getScratch();
+
+   const int idx = dst.getIndex(0);
+
+   switch (dst.getFile()) {
+   case TGSI_FILE_TEMPORARY:
+      return tData.acquire(idx, c);
+   case TGSI_FILE_TEMPORARY_ARRAY:
+      return getScratch();
+   case TGSI_FILE_PREDICATE:
+      return pData.acquire(idx, c);
+   case TGSI_FILE_ADDRESS:
+      return aData.acquire(idx, c);
+
+   case TGSI_FILE_OUTPUT:
+      if (prog->getType() == Program::TYPE_FRAGMENT)
+         return oData.acquire(idx, c);
+      // fall through
+   case TGSI_FILE_SYSTEM_VALUE:
+      return getScratch();
+
+   default:
+      assert(!"invalid dst file");
+      return NULL;
+   }
+}
+
+void
+Converter::storeDst(int d, int c, Value *val)
+{
+   const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
+
+   switch (tgsi.getSaturate()) {
+   case TGSI_SAT_NONE:
+      break;
+   case TGSI_SAT_ZERO_ONE:
+      mkOp1(OP_SAT, dstTy, val, val);
+      break;
+   case TGSI_SAT_MINUS_PLUS_ONE:
+      mkOp2(OP_MAX, dstTy, val, val, mkImm(-1.0f));
+      mkOp2(OP_MIN, dstTy, val, val, mkImm(+1.0f));
+      break;
+   default:
+      assert(!"invalid saturation mode");
+      break;
+   }
+
+   Value *ptr = dst.isIndirect(0) ?
+      fetchSrc(dst.getIndirect(0), 0, NULL) : NULL;
+
+   if (info->io.clipDistanceCount &&
+       dst.getFile() == TGSI_FILE_OUTPUT &&
+       info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_POSITION) {
+      mkMov(clipVtx[c], val);
+      val = clipVtx[c];
+   }
+
+   storeDst(dst, c, val, ptr);
+}
+
+void
+Converter::storeDst(const tgsi::Instruction::DstRegister dst, int c,
+                    Value *val, Value *ptr)
+{
+   const int idx = dst.getIndex(0);
+
+   switch (dst.getFile()) {
+   case TGSI_FILE_TEMPORARY:
+      tData.store(idx, c, ptr, val);
+      break;
+   case TGSI_FILE_TEMPORARY_ARRAY:
+      assert(dst.is2D() && dst.getIndex(1) < code->tempArrayCount);
+      lData[dst.getIndex(1)].store(idx, c, ptr, val);
+      break;
+   case TGSI_FILE_PREDICATE:
+      pData.store(idx, c, ptr, val);
+      break;
+   case TGSI_FILE_ADDRESS:
+      aData.store(idx, c, ptr, val);
+      break;
+
+   case TGSI_FILE_OUTPUT:
+      if (prog->getType() == Program::TYPE_FRAGMENT)
+         oData.store(idx, c, ptr, val);
+      else
+         mkStore(OP_EXPORT, TYPE_U32, dstToSym(dst, c), ptr, val);
+      break;
+
+   case TGSI_FILE_SYSTEM_VALUE:
+      assert(!ptr);
+      mkOp2(OP_WRSV, TYPE_U32, NULL, dstToSym(dst, c), val);
+      break;
+
+   default:
+      assert(!"invalid dst file");
+      break;
+   }
+}
+
+#define FOR_EACH_DST_ENABLED_CHANNEL(d, chan, inst) \
+   for (chan = 0; chan < 4; ++chan)                 \
+      if (!inst.getDst(d).isMasked(chan))
+
+Value *
+Converter::buildDot(int dim)
+{
+   assert(dim > 0);
+
+   Value *src0 = fetchSrc(0, 0), *src1 = fetchSrc(1, 0);
+   Value *dotp = getScratch();
+
+   mkOp2(OP_MUL, TYPE_F32, dotp, src0, src1);
+
+   for (int c = 1; c < dim; ++c) {
+      src0 = fetchSrc(0, c);
+      src1 = fetchSrc(1, c);
+      mkOp3(OP_MAD, TYPE_F32, dotp, src0, src1, dotp);
+   }
+   return dotp;
+}
+
+void
+Converter::insertConvergenceOps(BasicBlock *conv, BasicBlock *fork)
+{
+   FlowInstruction *join = new_FlowInstruction(func, OP_JOIN, NULL);
+   join->fixed = 1;
+   conv->insertHead(join);
+
+   fork->joinAt = new_FlowInstruction(func, OP_JOINAT, conv);
+   fork->insertBefore(fork->getExit(), fork->joinAt);
+}
+
+void
+Converter::setTexRS(TexInstruction *tex, unsigned int& s, int R, int S)
+{
+   unsigned rIdx = 0, sIdx = 0;
+
+   if (R >= 0)
+      rIdx = tgsi.getSrc(R).getIndex(0);
+   if (S >= 0)
+      sIdx = tgsi.getSrc(S).getIndex(0);
+
+   tex->setTexture(tgsi.getTexture(code, R), rIdx, sIdx);
+
+   if (tgsi.getSrc(R).isIndirect(0)) {
+      tex->tex.rIndirectSrc = s;
+      tex->setSrc(s++, fetchSrc(tgsi.getSrc(R).getIndirect(0), 0, NULL));
+   }
+   if (S >= 0 && tgsi.getSrc(S).isIndirect(0)) {
+      tex->tex.sIndirectSrc = s;
+      tex->setSrc(s++, fetchSrc(tgsi.getSrc(S).getIndirect(0), 0, NULL));
+   }
+}
+
+void
+Converter::handleTXQ(Value *dst0[4], enum TexQuery query)
+{
+   TexInstruction *tex = new_TexInstruction(func, OP_TXQ);
+   tex->tex.query = query;
+   unsigned int c, d;
+
+   for (d = 0, c = 0; c < 4; ++c) {
+      if (!dst0[c])
+         continue;
+      tex->tex.mask |= 1 << c;
+      tex->setDef(d++, dst0[c]);
+   }
+   tex->setSrc((c = 0), fetchSrc(0, 0)); // mip level
+
+   setTexRS(tex, c, 1, -1);
+
+   bb->insertTail(tex);
+}
+
+void
+Converter::loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask)
+{
+   Value *proj = fetchSrc(0, 3);
+   Instruction *insn = proj->getUniqueInsn();
+   int c;
+
+   if (insn->op == OP_PINTERP) {
+      bb->insertTail(insn = insn->clone(true));
+      insn->op = OP_LINTERP;
+      insn->setInterpolate(NV50_IR_INTERP_LINEAR | insn->getSampleMode());
+      insn->setSrc(1, NULL);
+      proj = insn->getDef(0);
+   }
+   proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), proj);
+
+   for (c = 0; c < 4; ++c) {
+      if (!(mask & (1 << c)))
+         continue;
+      if ((insn = src[c]->getUniqueInsn())->op != OP_PINTERP)
+         continue;
+      mask &= ~(1 << c);
+
+      bb->insertTail(insn = insn->clone(true));
+      insn->setInterpolate(NV50_IR_INTERP_PERSPECTIVE | insn->getSampleMode());
+      insn->setSrc(1, proj);
+      dst[c] = insn->getDef(0);
+   }
+   if (!mask)
+      return;
+
+   proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), fetchSrc(0, 3));
+
+   for (c = 0; c < 4; ++c)
+      if (mask & (1 << c))
+         dst[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), src[c], proj);
+}
+
+// order of nv50 ir sources: x y z layer lod/bias shadow
+// order of TGSI TEX sources: x y z layer shadow lod/bias
+//  lowering will finally set the hw specific order (like array first on nvc0)
+void
+Converter::handleTEX(Value *dst[4], int R, int S, int L, int C, int Dx, int Dy)
+{
+   Value *val;
+   Value *arg[4], *src[8];
+   Value *lod = NULL, *shd = NULL;
+   unsigned int s, c, d;
+   TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
+
+   TexInstruction::Target tgt = tgsi.getTexture(code, R);
+
+   for (s = 0; s < tgt.getArgCount(); ++s)
+      arg[s] = src[s] = fetchSrc(0, s);
+
+   if (texi->op == OP_TXL || texi->op == OP_TXB)
+      lod = fetchSrc(L >> 4, L & 3);
+
+   if (C == 0x0f)
+      C = 0x00 | MAX2(tgt.getArgCount(), 2); // guess DC src
+
+   if (tgt.isShadow())
+      shd = fetchSrc(C >> 4, C & 3);
+
+   if (texi->op == OP_TXD) {
+      for (c = 0; c < tgt.getDim(); ++c) {
+         texi->dPdx[c] = fetchSrc(Dx >> 4, (Dx & 3) + c);
+         texi->dPdy[c] = fetchSrc(Dy >> 4, (Dy & 3) + c);
+      }
+   }
+
+   // cube textures don't care about projection value, it's divided out
+   if (tgsi.getOpcode() == TGSI_OPCODE_TXP && !tgt.isCube() && !tgt.isArray()) {
+      unsigned int n = tgt.getDim();
+      if (shd) {
+         arg[n] = shd;
+         ++n;
+         assert(tgt.getDim() == tgt.getArgCount());
+      }
+      loadProjTexCoords(src, arg, (1 << n) - 1);
+      if (shd)
+         shd = src[n - 1];
+   }
+
+   if (tgt.isCube()) {
+      for (c = 0; c < 3; ++c)
+         src[c] = mkOp1v(OP_ABS, TYPE_F32, getSSA(), arg[c]);
+      val = getScratch();
+      mkOp2(OP_MAX, TYPE_F32, val, src[0], src[1]);
+      mkOp2(OP_MAX, TYPE_F32, val, src[2], val);
+      mkOp1(OP_RCP, TYPE_F32, val, val);
+      for (c = 0; c < 3; ++c)
+         src[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), arg[c], val);
+   }
+
+   for (c = 0, d = 0; c < 4; ++c) {
+      if (dst[c]) {
+         texi->setDef(d++, dst[c]);
+         texi->tex.mask |= 1 << c;
+      } else {
+         // NOTE: maybe hook up def too, for CSE
+      }
+   }
+   for (s = 0; s < tgt.getArgCount(); ++s)
+      texi->setSrc(s, src[s]);
+   if (lod)
+      texi->setSrc(s++, lod);
+   if (shd)
+      texi->setSrc(s++, shd);
+
+   setTexRS(texi, s, R, S);
+
+   if (tgsi.getOpcode() == TGSI_OPCODE_SAMPLE_C_LZ)
+      texi->tex.levelZero = true;
+
+   bb->insertTail(texi);
+}
+
+// 1st source: xyz = coordinates, w = lod
+// 2nd source: offset
+void
+Converter::handleTXF(Value *dst[4], int R)
+{
+   TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
+   unsigned int c, d, s;
+
+   texi->tex.target = tgsi.getTexture(code, R);
+
+   for (c = 0, d = 0; c < 4; ++c) {
+      if (dst[c]) {
+         texi->setDef(d++, dst[c]);
+         texi->tex.mask |= 1 << c;
+      }
+   }
+   for (c = 0; c < texi->tex.target.getArgCount(); ++c)
+      texi->setSrc(c, fetchSrc(0, c));
+   texi->setSrc(c++, fetchSrc(0, 3)); // lod
+
+   setTexRS(texi, c, R, -1);
+
+   for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
+      for (c = 0; c < 3; ++c) {
+         texi->tex.offset[s][c] = tgsi.getTexOffset(s).getValueU32(c, info);
+         if (texi->tex.offset[s][c])
+            texi->tex.useOffsets = s + 1;
+      }
+   }
+
+   bb->insertTail(texi);
+}
+
+void
+Converter::handleLIT(Value *dst0[4])
+{
+   Value *val0 = NULL;
+   unsigned int mask = tgsi.getDst(0).getMask();
+
+   if (mask & (1 << 0))
+      loadImm(dst0[0], 1.0f);
+
+   if (mask & (1 << 3))
+      loadImm(dst0[3], 1.0f);
+
+   if (mask & (3 << 1)) {
+      val0 = getScratch();
+      mkOp2(OP_MAX, TYPE_F32, val0, fetchSrc(0, 0), zero);
+      if (mask & (1 << 1))
+         mkMov(dst0[1], val0);
+   }
+
+   if (mask & (1 << 2)) {
+      Value *src1 = fetchSrc(0, 1), *src3 = fetchSrc(0, 3);
+      Value *val1 = getScratch(), *val3 = getScratch();
+
+      Value *pos128 = loadImm(NULL, +127.999999f);
+      Value *neg128 = loadImm(NULL, -127.999999f);
+
+      mkOp2(OP_MAX, TYPE_F32, val1, src1, zero);
+      mkOp2(OP_MAX, TYPE_F32, val3, src3, neg128);
+      mkOp2(OP_MIN, TYPE_F32, val3, val3, pos128);
+      mkOp2(OP_POW, TYPE_F32, val3, val1, val3);
+
+      mkCmp(OP_SLCT, CC_GT, TYPE_F32, dst0[2], val3, zero, val0);
+   }
+}
+
+bool
+Converter::isEndOfSubroutine(uint ip)
+{
+   assert(ip < code->scan.num_instructions);
+   tgsi::Instruction insn(&code->insns[ip]);
+   return (insn.getOpcode() == TGSI_OPCODE_END ||
+           insn.getOpcode() == TGSI_OPCODE_ENDSUB ||
+           // does END occur at end of main or the very end ?
+           insn.getOpcode() == TGSI_OPCODE_BGNSUB);
+}
+
+bool
+Converter::handleInstruction(const struct tgsi_full_instruction *insn)
+{
+   Value *dst0[4], *rDst0[4];
+   Value *src0, *src1, *src2;
+   Value *val0, *val1;
+   int c;
+
+   tgsi = tgsi::Instruction(insn);
+
+   bool useScratchDst = tgsi.checkDstSrcAliasing();
+
+   operation op = tgsi.getOP();
+   dstTy = tgsi.inferDstType();
+   srcTy = tgsi.inferSrcType();
+
+   unsigned int mask = tgsi.dstCount() ? tgsi.getDst(0).getMask() : 0;
+
+   if (tgsi.dstCount()) {
+      for (c = 0; c < 4; ++c) {
+         rDst0[c] = acquireDst(0, c);
+         dst0[c] = (useScratchDst && rDst0[c]) ? getScratch() : rDst0[c];
+      }
+   }
+
+   switch (tgsi.getOpcode()) {
+   case TGSI_OPCODE_ADD:
+   case TGSI_OPCODE_UADD:
+   case TGSI_OPCODE_AND:
+   case TGSI_OPCODE_DIV:
+   case TGSI_OPCODE_IDIV:
+   case TGSI_OPCODE_UDIV:
+   case TGSI_OPCODE_MAX:
+   case TGSI_OPCODE_MIN:
+   case TGSI_OPCODE_IMAX:
+   case TGSI_OPCODE_IMIN:
+   case TGSI_OPCODE_UMAX:
+   case TGSI_OPCODE_UMIN:
+   case TGSI_OPCODE_MOD:
+   case TGSI_OPCODE_UMOD:
+   case TGSI_OPCODE_MUL:
+   case TGSI_OPCODE_UMUL:
+   case TGSI_OPCODE_OR:
+   case TGSI_OPCODE_POW:
+   case TGSI_OPCODE_SHL:
+   case TGSI_OPCODE_ISHR:
+   case TGSI_OPCODE_USHR:
+   case TGSI_OPCODE_SUB:
+   case TGSI_OPCODE_XOR:
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
+         src0 = fetchSrc(0, c);
+         src1 = fetchSrc(1, c);
+         mkOp2(op, dstTy, dst0[c], src0, src1);
+      }
+      break;
+   case TGSI_OPCODE_MAD:
+   case TGSI_OPCODE_UMAD:
+   case TGSI_OPCODE_SAD:
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
+         src0 = fetchSrc(0, c);
+         src1 = fetchSrc(1, c);
+         src2 = fetchSrc(2, c);
+         mkOp3(op, dstTy, dst0[c], src0, src1, src2);
+      }      
+      break;
+   case TGSI_OPCODE_MOV:
+   case TGSI_OPCODE_ABS:
+   case TGSI_OPCODE_CEIL:
+   case TGSI_OPCODE_FLR:
+   case TGSI_OPCODE_TRUNC:
+   case TGSI_OPCODE_RCP:
+   case TGSI_OPCODE_INEG:
+   case TGSI_OPCODE_NOT:
+   case TGSI_OPCODE_DDX:
+   case TGSI_OPCODE_DDY:
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
+         mkOp1(op, dstTy, dst0[c], fetchSrc(0, c));
+      break;
+   case TGSI_OPCODE_RSQ:
+      src0 = fetchSrc(0, 0);
+      val0 = getScratch();
+      mkOp1(OP_ABS, TYPE_F32, val0, src0);
+      mkOp1(OP_RSQ, TYPE_F32, val0, val0);
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
+         mkMov(dst0[c], val0);
+      break;
+   case TGSI_OPCODE_ARL:
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
+         src0 = fetchSrc(0, c);
+         mkCvt(OP_CVT, TYPE_S32, dst0[c], TYPE_F32, src0)->rnd = ROUND_M;
+         mkOp2(OP_SHL, TYPE_U32, dst0[c], dst0[c], mkImm(4));
+      }
+      break;
+   case TGSI_OPCODE_UARL:
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
+         mkOp2(OP_SHL, TYPE_U32, dst0[c], fetchSrc(0, c), mkImm(4));
+      break;
+   case TGSI_OPCODE_EX2:
+   case TGSI_OPCODE_LG2:
+      val0 = mkOp1(op, TYPE_F32, getScratch(), fetchSrc(0, 0))->getDef(0);
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
+         mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
+      break;
+   case TGSI_OPCODE_COS:
+   case TGSI_OPCODE_SIN:
+      val0 = getScratch();
+      if (mask & 7) {
+         mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 0));
+         mkOp1(op, TYPE_F32, val0, val0);
+         for (c = 0; c < 3; ++c)
+            if (dst0[c])
+               mkMov(dst0[c], val0);
+      }
+      if (dst0[3]) {
+         mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 3));
+         mkOp1(op, TYPE_F32, dst0[3], val0);
+      }
+      break;
+   case TGSI_OPCODE_SCS:
+      if (mask & 3) {
+         val0 = mkOp1v(OP_PRESIN, TYPE_F32, getSSA(), fetchSrc(0, 0));
+         if (dst0[0])
+            mkOp1(OP_COS, TYPE_F32, dst0[0], val0);
+         if (dst0[1])
+            mkOp1(OP_SIN, TYPE_F32, dst0[1], val0);
+      }
+      if (dst0[2])
+         loadImm(dst0[2], 0.0f);
+      if (dst0[3])
+         loadImm(dst0[3], 1.0f);
+      break;
+   case TGSI_OPCODE_EXP:
+      src0 = fetchSrc(0, 0);
+      val0 = mkOp1v(OP_FLOOR, TYPE_F32, getSSA(), src0);
+      if (dst0[1])
+         mkOp2(OP_SUB, TYPE_F32, dst0[1], src0, val0);
+      if (dst0[0])
+         mkOp1(OP_EX2, TYPE_F32, dst0[0], val0);
+      if (dst0[2])
+         mkOp1(OP_EX2, TYPE_F32, dst0[2], src0);
+      if (dst0[3])
+         loadImm(dst0[3], 1.0f);
+      break;
+   case TGSI_OPCODE_LOG:
+      src0 = mkOp1v(OP_ABS, TYPE_F32, getSSA(), fetchSrc(0, 0));
+      val0 = mkOp1v(OP_LG2, TYPE_F32, dst0[2] ? dst0[2] : getSSA(), src0);
+      if (dst0[0] || dst0[1])
+         val1 = mkOp1v(OP_FLOOR, TYPE_F32, dst0[0] ? dst0[0] : getSSA(), val0);
+      if (dst0[1]) {
+         mkOp1(OP_EX2, TYPE_F32, dst0[1], val1);
+         mkOp1(OP_RCP, TYPE_F32, dst0[1], dst0[1]);
+         mkOp2(OP_MUL, TYPE_F32, dst0[1], dst0[1], src0);
+      }
+      if (dst0[3])
+         loadImm(dst0[3], 1.0f);
+      break;
+   case TGSI_OPCODE_DP2:
+      val0 = buildDot(2);
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
+         mkMov(dst0[c], val0);
+      break;
+   case TGSI_OPCODE_DP3:
+      val0 = buildDot(3);
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
+         mkMov(dst0[c], val0);
+      break;
+   case TGSI_OPCODE_DP4:
+      val0 = buildDot(4);
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
+         mkMov(dst0[c], val0);
+      break;
+   case TGSI_OPCODE_DPH:
+      val0 = buildDot(3);
+      src1 = fetchSrc(1, 3);
+      mkOp2(OP_ADD, TYPE_F32, val0, val0, src1);
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
+         mkMov(dst0[c], val0);
+      break;
+   case TGSI_OPCODE_DST:
+      if (dst0[0])
+         loadImm(dst0[0], 1.0f);
+      if (dst0[1]) {
+         src0 = fetchSrc(0, 1);
+         src1 = fetchSrc(1, 1);
+         mkOp2(OP_MUL, TYPE_F32, dst0[1], src0, src1);
+      }
+      if (dst0[2])
+         mkMov(dst0[2], fetchSrc(0, 2));
+      if (dst0[3])
+         mkMov(dst0[3], fetchSrc(1, 3));
+      break;
+   case TGSI_OPCODE_LRP:
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
+         src0 = fetchSrc(0, c);
+         src1 = fetchSrc(1, c);
+         src2 = fetchSrc(2, c);
+         mkOp3(OP_MAD, TYPE_F32, dst0[c],
+               mkOp2v(OP_SUB, TYPE_F32, getSSA(), src1, src2), src0, src2);
+      }
+      break;
+   case TGSI_OPCODE_LIT:
+      handleLIT(dst0);
+      break;
+   case TGSI_OPCODE_XPD:
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
+         if (c < 3) {
+            val0 = getSSA();
+            src0 = fetchSrc(1, (c + 1) % 3);
+            src1 = fetchSrc(0, (c + 2) % 3);
+            mkOp2(OP_MUL, TYPE_F32, val0, src0, src1);
+            mkOp1(OP_NEG, TYPE_F32, val0, val0);
+
+            src0 = fetchSrc(0, (c + 1) % 3);
+            src1 = fetchSrc(1, (c + 2) % 3);
+            mkOp3(OP_MAD, TYPE_F32, dst0[c], src0, src1, val0);
+         } else {
+            loadImm(dst0[c], 1.0f);
+         }
+      }
+      break;
+   case TGSI_OPCODE_SSG:
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
+         src0 = fetchSrc(0, c);
+         val0 = getScratch();
+         val1 = getScratch();
+         mkCmp(OP_SET, CC_GT, TYPE_F32, val0, src0, zero);
+         mkCmp(OP_SET, CC_LT, TYPE_F32, val1, src0, zero);
+         mkOp2(OP_SUB, TYPE_F32, dst0[c], val0, val1);
+      }
+      break;
+   case TGSI_OPCODE_UCMP:
+   case TGSI_OPCODE_CMP:
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
+         src0 = fetchSrc(0, c);
+         src1 = fetchSrc(1, c);
+         src2 = fetchSrc(2, c);
+         if (src1 == src2)
+            mkMov(dst0[c], src1);
+         else
+            mkCmp(OP_SLCT, (srcTy == TYPE_F32) ? CC_LT : CC_NE,
+                  srcTy, dst0[c], src1, src2, src0);
+      }
+      break;
+   case TGSI_OPCODE_FRC:
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
+         src0 = fetchSrc(0, c);
+         val0 = getScratch();
+         mkOp1(OP_FLOOR, TYPE_F32, val0, src0);
+         mkOp2(OP_SUB, TYPE_F32, dst0[c], src0, val0);
+      }
+      break;
+   case TGSI_OPCODE_ROUND:
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
+         mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F32, fetchSrc(0, c))
+         ->rnd = ROUND_NI;
+      break;
+   case TGSI_OPCODE_CLAMP:
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
+         src0 = fetchSrc(0, c);
+         src1 = fetchSrc(1, c);
+         src2 = fetchSrc(2, c);
+         val0 = getScratch();
+         mkOp2(OP_MIN, TYPE_F32, val0, src0, src1);
+         mkOp2(OP_MAX, TYPE_F32, dst0[c], val0, src2);
+      }
+      break;
+   case TGSI_OPCODE_SLT:
+   case TGSI_OPCODE_SGE:
+   case TGSI_OPCODE_SEQ:
+   case TGSI_OPCODE_SFL:
+   case TGSI_OPCODE_SGT:
+   case TGSI_OPCODE_SLE:
+   case TGSI_OPCODE_SNE:
+   case TGSI_OPCODE_STR:
+   case TGSI_OPCODE_ISGE:
+   case TGSI_OPCODE_ISLT:
+   case TGSI_OPCODE_USEQ:
+   case TGSI_OPCODE_USGE:
+   case TGSI_OPCODE_USLT:
+   case TGSI_OPCODE_USNE:
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
+         src0 = fetchSrc(0, c);
+         src1 = fetchSrc(1, c);
+         mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], src0, src1);
+      }
+      break;
+   case TGSI_OPCODE_KIL:
+      val0 = new_LValue(func, FILE_PREDICATE);
+      for (c = 0; c < 4; ++c) {
+         mkCmp(OP_SET, CC_LT, TYPE_F32, val0, fetchSrc(0, c), zero);
+         mkOp(OP_DISCARD, TYPE_NONE, NULL)->setPredicate(CC_P, val0);
+      }
+      break;
+   case TGSI_OPCODE_KILP:
+      mkOp(OP_DISCARD, TYPE_NONE, NULL);
+      break;
+   case TGSI_OPCODE_TEX:
+   case TGSI_OPCODE_TXB:
+   case TGSI_OPCODE_TXL:
+   case TGSI_OPCODE_TXP:
+      //              R  S     L     C    Dx    Dy
+      handleTEX(dst0, 1, 1, 0x03, 0x0f, 0x00, 0x00);
+      break;
+   case TGSI_OPCODE_TXD:
+      handleTEX(dst0, 3, 3, 0x03, 0x0f, 0x10, 0x20);
+      break;
+   case TGSI_OPCODE_SAMPLE:
+   case TGSI_OPCODE_SAMPLE_B:
+   case TGSI_OPCODE_SAMPLE_D:
+   case TGSI_OPCODE_SAMPLE_L:
+   case TGSI_OPCODE_SAMPLE_C:
+   case TGSI_OPCODE_SAMPLE_C_LZ:
+      handleTEX(dst0, 1, 2, 0x30, 0x31, 0x40, 0x50);
+      break;
+   case TGSI_OPCODE_TXF:
+   case TGSI_OPCODE_LOAD:
+      handleTXF(dst0, 1);
+      break;
+   case TGSI_OPCODE_TXQ:
+   case TGSI_OPCODE_RESINFO:
+      handleTXQ(dst0, TXQ_DIMS);
+      break;
+   case TGSI_OPCODE_F2I:
+   case TGSI_OPCODE_F2U:
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
+         mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c))->rnd = ROUND_Z;
+      break;
+   case TGSI_OPCODE_I2F:
+   case TGSI_OPCODE_U2F:
+      FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
+         mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c));
+      break;
+   case TGSI_OPCODE_EMIT:
+   case TGSI_OPCODE_ENDPRIM:
+      // get vertex stream if specified (must be immediate)
+      src0 = tgsi.srcCount() ?
+         mkImm(tgsi.getSrc(0).getValueU32(0, info)) : zero;
+      mkOp1(op, TYPE_U32, NULL, src0)->fixed = 1;
+      break;
+   case TGSI_OPCODE_IF:
+   {
+      BasicBlock *ifBB = new BasicBlock(func);
+
+      bb->cfg.attach(&ifBB->cfg, Graph::Edge::TREE);
+      condBBs.push(bb);
+      joinBBs.push(bb);
+
+      mkFlow(OP_BRA, NULL, CC_NOT_P, fetchSrc(0, 0));
+
+      setPosition(ifBB, true);
+   }
+      break;
+   case TGSI_OPCODE_ELSE:
+   {
+      BasicBlock *elseBB = new BasicBlock(func);
+      BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
+
+      forkBB->cfg.attach(&elseBB->cfg, Graph::Edge::TREE);
+      condBBs.push(bb);
+
+      forkBB->getExit()->asFlow()->target.bb = elseBB;
+      if (!bb->isTerminated())
+         mkFlow(OP_BRA, NULL, CC_ALWAYS, NULL);
+
+      setPosition(elseBB, true);
+   }
+      break;
+   case TGSI_OPCODE_ENDIF:
+   {
+      BasicBlock *convBB = new BasicBlock(func);
+      BasicBlock *prevBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
+      BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(joinBBs.pop().u.p);
+
+      if (!bb->isTerminated()) {
+         // we only want join if none of the clauses ended with CONT/BREAK/RET
+         if (prevBB->getExit()->op == OP_BRA && joinBBs.getSize() < 6)
+            insertConvergenceOps(convBB, forkBB);
+         mkFlow(OP_BRA, convBB, CC_ALWAYS, NULL);
+         bb->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
+      }
+
+      if (prevBB->getExit()->op == OP_BRA) {
+         prevBB->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
+         prevBB->getExit()->asFlow()->target.bb = convBB;
+      }
+      setPosition(convBB, true);
+   }
+      break;
+   case TGSI_OPCODE_BGNLOOP:
+   {
+      BasicBlock *lbgnBB = new BasicBlock(func);
+      BasicBlock *lbrkBB = new BasicBlock(func);
+
+      loopBBs.push(lbgnBB);
+      breakBBs.push(lbrkBB);
+      if (loopBBs.getSize() > func->loopNestingBound)
+         func->loopNestingBound++;
+
+      mkFlow(OP_PREBREAK, lbrkBB, CC_ALWAYS, NULL);
+
+      bb->cfg.attach(&lbgnBB->cfg, Graph::Edge::TREE);
+      setPosition(lbgnBB, true);
+      mkFlow(OP_PRECONT, lbgnBB, CC_ALWAYS, NULL);
+   }
+      break;
+   case TGSI_OPCODE_ENDLOOP:
+   {
+      BasicBlock *loopBB = reinterpret_cast<BasicBlock *>(loopBBs.pop().u.p);
+
+      if (!bb->isTerminated()) {
+         mkFlow(OP_CONT, loopBB, CC_ALWAYS, NULL);
+         bb->cfg.attach(&loopBB->cfg, Graph::Edge::BACK);
+      }
+      setPosition(reinterpret_cast<BasicBlock *>(breakBBs.pop().u.p), true);
+   }
+      break;
+   case TGSI_OPCODE_BRK:
+   {
+      if (bb->isTerminated())
+         break;
+      BasicBlock *brkBB = reinterpret_cast<BasicBlock *>(breakBBs.peek().u.p);
+      mkFlow(OP_BREAK, brkBB, CC_ALWAYS, NULL);
+      bb->cfg.attach(&brkBB->cfg, Graph::Edge::CROSS);
+   }
+      break;
+   case TGSI_OPCODE_CONT:
+   {
+      if (bb->isTerminated())
+         break;
+      BasicBlock *contBB = reinterpret_cast<BasicBlock *>(loopBBs.peek().u.p);
+      mkFlow(OP_CONT, contBB, CC_ALWAYS, NULL);
+      contBB->explicitCont = true;
+      bb->cfg.attach(&contBB->cfg, Graph::Edge::BACK);
+   }
+      break;
+   case TGSI_OPCODE_BGNSUB:
+   {
+      if (!retIPs.getSize()) {
+         // end of main function
+         ip = code->scan.num_instructions - 2; // goto END
+         return true;
+      }
+      BasicBlock *entry = new BasicBlock(func);
+      BasicBlock *leave = new BasicBlock(func);
+      entryBBs.push(entry);
+      leaveBBs.push(leave);
+      bb->cfg.attach(&entry->cfg, Graph::Edge::TREE);
+      setPosition(entry, true);
+   }
+      return true;
+   case TGSI_OPCODE_ENDSUB:
+   {
+      BasicBlock *leave = reinterpret_cast<BasicBlock *>(leaveBBs.pop().u.p);
+      entryBBs.pop();
+      bb->cfg.attach(&leave->cfg, Graph::Edge::TREE);
+      setPosition(leave, true);
+      ip = retIPs.pop().u.u;
+   }
+      return true;
+   case TGSI_OPCODE_CAL:
+      // we don't have function declarations, so inline everything
+      retIPs.push(ip);
+      ip = code->subroutines[tgsi.getLabel()].pc - 1; // +1 after return
+      return true;
+   case TGSI_OPCODE_RET:
+   {
+      if (bb->isTerminated())
+         return true;
+      BasicBlock *entry = reinterpret_cast<BasicBlock *>(entryBBs.peek().u.p);
+      BasicBlock *leave = reinterpret_cast<BasicBlock *>(leaveBBs.peek().u.p);
+      if (!isEndOfSubroutine(ip + 1)) {
+         // insert a PRERET at the entry if this is an early return
+         FlowInstruction *preRet = new_FlowInstruction(func, OP_PRERET, leave);
+         preRet->fixed = 1;
+         entry->insertHead(preRet);
+         bb->cfg.attach(&leave->cfg, Graph::Edge::CROSS);
+      }
+      // everything inlined so RET serves only to wrap up the stack
+      if (entry->getEntry() && entry->getEntry()->op == OP_PRERET)
+         mkFlow(OP_RET, NULL, CC_ALWAYS, NULL)->fixed = 1;
+   }
+      break;
+   case TGSI_OPCODE_END:
+   {
+      // attach and generate epilogue code
+      BasicBlock *epilogue = reinterpret_cast<BasicBlock *>(leaveBBs.pop().u.p);
+      entryBBs.pop();
+      bb->cfg.attach(&epilogue->cfg, Graph::Edge::TREE);
+      setPosition(epilogue, true);
+      if (prog->getType() == Program::TYPE_FRAGMENT)
+         exportOutputs();
+      if (info->io.clipDistanceCount)
+         handleUserClipPlanes();
+      mkOp(OP_EXIT, TYPE_NONE, NULL)->terminator = 1;
+   }
+      break;
+   case TGSI_OPCODE_SWITCH:
+   case TGSI_OPCODE_CASE:
+      ERROR("switch/case opcode encountered, should have been lowered\n");
+      abort();
+      break;
+   default:
+      ERROR("unhandled TGSI opcode: %u\n", tgsi.getOpcode());
+      assert(0);
+      break;
+   }
+
+   if (tgsi.dstCount()) {
+      for (c = 0; c < 4; ++c) {
+         if (!dst0[c])
+            continue;
+         if (dst0[c] != rDst0[c])
+            mkMov(rDst0[c], dst0[c]);
+         storeDst(0, c, rDst0[c]);
+      }
+   }
+   vtxBaseValid = 0;
+
+   return true;
+}
+
+void
+Converter::handleUserClipPlanes()
+{
+   Value *res[8];
+   int i, c;
+
+   for (c = 0; c < 4; ++c) {
+      for (i = 0; i < info->io.clipDistanceCount; ++i) {
+         Value *ucp;
+         ucp = mkLoad(TYPE_F32, mkSymbol(FILE_MEMORY_CONST, 15, TYPE_F32,
+                                         i * 16 + c * 4), NULL);
+         if (c == 0)
+            res[i] = mkOp2v(OP_MUL, TYPE_F32, getScratch(), clipVtx[c], ucp);
+         else
+            mkOp3(OP_MAD, TYPE_F32, res[i], clipVtx[c], ucp, res[i]);
+      }
+   }
+
+   for (i = 0; i < info->io.clipDistanceCount; ++i)
+      mkOp2(OP_WRSV, TYPE_F32, NULL, mkSysVal(SV_CLIP_DISTANCE, i), res[i]);
+}
+
+void
+Converter::exportOutputs()
+{
+   for (unsigned int i = 0; i < info->numOutputs; ++i) {
+      for (unsigned int c = 0; c < 4; ++c) {
+         if (!oData.exists(i, c))
+            continue;
+         Symbol *sym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32,
+                                info->out[i].slot[c] * 4);
+         Value *val = oData.load(i, c, NULL);
+         if (val)
+            mkStore(OP_EXPORT, TYPE_F32, sym, NULL, val);
+      }
+   }
+}
+
+Converter::Converter(Program *ir, const tgsi::Source *src)
+   : code(src),
+     tgsi(NULL),
+     tData(this), aData(this), pData(this), oData(this)
+{
+   prog = ir;
+   info = code->info;
+
+   DataFile tFile = code->mainTempsInLMem ? FILE_MEMORY_LOCAL : FILE_GPR;
+
+   tData.setup(0, code->fileSize(TGSI_FILE_TEMPORARY), 4, 4, tFile);
+   pData.setup(0, code->fileSize(TGSI_FILE_PREDICATE), 4, 4, FILE_PREDICATE);
+   aData.setup(0, code->fileSize(TGSI_FILE_ADDRESS), 4, 4, FILE_ADDRESS);
+   oData.setup(0, code->fileSize(TGSI_FILE_OUTPUT), 4, 4, FILE_GPR);
+
+   lData = NULL;
+   iData = NULL;
+
+   zero = mkImm((uint32_t)0);
+
+   vtxBaseValid = 0;
+}
+
+Converter::~Converter()
+{
+   if (lData)
+      delete[] lData;
+   if (iData)
+      delete[] iData;
+}
+
+bool
+Converter::run()
+{
+   BasicBlock *entry = new BasicBlock(prog->main);
+   BasicBlock *leave = new BasicBlock(prog->main);
+
+   if (code->tempArrayCount && !lData) {
+      uint32_t volume = 0;
+      lData = new DataArray[code->tempArrayCount];
+      if (!lData)
+         return false;
+      for (int i = 0; i < code->tempArrayCount; ++i) {
+         int len = code->tempArrays[i].u32 >> 2;
+         int dim = code->tempArrays[i].u32 & 3;
+         lData[i].setParent(this);
+         lData[i].setup(volume, len, dim, 4, FILE_MEMORY_LOCAL);
+         volume += (len * dim * 4 + 0xf) & ~0xf;
+      }
+   }
+   if (code->immdArrayCount && !iData) {
+      uint32_t volume = 0;
+      iData = new DataArray[code->immdArrayCount];
+      if (!iData)
+         return false;
+      for (int i = 0; i < code->immdArrayCount; ++i) {
+         int len = code->immdArrays[i].u32 >> 2;
+         int dim = code->immdArrays[i].u32 & 3;
+         iData[i].setParent(this);
+         iData[i].setup(volume, len, dim, 4, FILE_MEMORY_CONST, 14);
+         volume += (len * dim * 4 + 0xf) & ~0xf;
+      }
+   }
+
+   prog->main->setEntry(entry);
+   prog->main->setExit(leave);
+
+   setPosition(entry, true);
+   entryBBs.push(entry);
+   leaveBBs.push(leave);
+
+   if (info->io.clipDistanceCount) {
+      for (int c = 0; c < 4; ++c)
+         clipVtx[c] = getScratch();
+   }
+
+   if (prog->getType() == Program::TYPE_FRAGMENT) {
+      Symbol *sv = mkSysVal(SV_POSITION, 3);
+      fragCoord[3] = mkOp1v(OP_RDSV, TYPE_F32, getSSA(), sv);
+      mkOp1(OP_RCP, TYPE_F32, fragCoord[3], fragCoord[3]);
+   }
+
+   for (ip = 0; ip < code->scan.num_instructions; ++ip) {
+      if (!handleInstruction(&code->insns[ip]))
+         return false;
+   }
+   return true;
+}
+
+} // unnamed namespace
+
+namespace nv50_ir {
+
+bool
+Program::makeFromTGSI(struct nv50_ir_prog_info *info)
+{
+   tgsi::Source src(info);
+   if (!src.scanSource())
+      return false;
+
+   Converter builder(this, &src);
+   return builder.run();
+}
+
+} // namespace nv50_ir
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_graph.cpp b/src/gallium/drivers/nv50/codegen/nv50_ir_graph.cpp
new file mode 100644
index 00000000000..08075751d14
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_graph.cpp
@@ -0,0 +1,381 @@
+
+#include "nv50_ir_graph.h"
+
+namespace nv50_ir {
+
+Graph::Graph()
+{
+   root = NULL;
+   size = 0;
+   sequence = 0;
+}
+
+Graph::~Graph()
+{
+   Iterator *iter = this->safeIteratorDFS();
+
+   for (; !iter->end(); iter->next())
+      reinterpret_cast<Node *>(iter->get())->cut();
+
+   putIterator(iter);
+}
+
+void Graph::insert(Node *node)
+{
+   if (!root) {
+      root = node;
+      size = 1;
+      node->graph = this;
+   } else {
+      root->attach(node, Edge::TREE);
+   }
+}
+
+void Graph::Edge::unlink()
+{
+   if (origin) {
+      prev[0]->next[0] = next[0];
+      next[0]->prev[0] = prev[0];
+      if (origin->out == this)
+         origin->out = (next[0] == this) ? NULL : next[0];
+
+      --origin->outCount;
+   }
+   if (target) {
+      prev[1]->next[1] = next[1];
+      next[1]->prev[1] = prev[1];
+      if (target->in == this)
+         target->in = (next[1] == this) ? NULL : next[1];
+
+      --target->inCount;
+   }
+}
+
+const char *Graph::Edge::typeStr() const
+{
+   switch (type) {
+   case TREE:    return "tree";
+   case FORWARD: return "forward";
+   case BACK:    return "back";
+   case CROSS:   return "cross";
+   case DUMMY:   return "dummy";
+   case UNKNOWN:
+   default:
+      return "unk";
+   }
+}
+
+Graph::Node::Node(void *priv) : data(priv),
+                                in(0), out(0), graph(0),
+                                visited(0),
+                                inCount(0), outCount(0)
+{
+   // nothing to do
+}
+
+void Graph::Node::attach(Node *node, Edge::Type kind)
+{
+   Edge *edge = new Edge(this, node, kind);
+
+   // insert head
+   if (this->out) {
+      edge->next[0] = this->out;
+      edge->prev[0] = this->out->prev[0];
+      edge->prev[0]->next[0] = edge;
+      this->out->prev[0] = edge;
+   }
+   this->out = edge;
+
+   if (node->in) {
+      edge->next[1] = node->in;
+      edge->prev[1] = node->in->prev[1];
+      edge->prev[1]->next[1] = edge;
+      node->in->prev[1] = edge;
+   }
+   node->in = edge;
+
+   ++this->outCount;
+   ++node->inCount;
+
+   assert(this->graph);
+   if (!node->graph) {
+      node->graph = this->graph;
+      ++node->graph->size;
+   }
+
+   if (kind == Edge::UNKNOWN)
+      graph->classifyEdges();
+}
+
+bool Graph::Node::detach(Graph::Node *node)
+{
+   EdgeIterator ei = this->outgoing();
+   for (; !ei.end(); ei.next())
+      if (ei.getNode() == node)
+         break;
+   if (ei.end()) {
+      ERROR("no such node attached\n");
+      return false;
+   }
+   delete ei.getEdge();
+   return true;
+}
+
+// Cut a node from the graph, deleting all attached edges.
+void Graph::Node::cut()
+{
+   if (!graph || (!in && !out))
+      return;
+
+   while (out)
+      delete out;
+   while (in)
+      delete in;
+
+   if (graph->root == this)
+      graph->root = NULL;
+}
+
+Graph::Edge::Edge(Node *org, Node *tgt, Type kind)
+{
+   target = tgt;
+   origin = org;
+   type = kind;
+
+   next[0] = next[1] = this;
+   prev[0] = prev[1] = this;
+}
+
+bool
+Graph::Node::reachableBy(Node *node, Node *term)
+{
+   Stack stack;
+   Node *pos;
+   const int seq = graph->nextSequence();
+
+   stack.push(node);
+
+   while (stack.getSize()) {
+      pos = reinterpret_cast<Node *>(stack.pop().u.p);
+
+      if (pos == this)
+         return true;
+      if (pos == term)
+         continue;
+
+      for (EdgeIterator ei = pos->outgoing(); !ei.end(); ei.next()) {
+         if (ei.getType() == Edge::BACK || ei.getType() == Edge::DUMMY)
+            continue;
+         if (ei.getNode()->visit(seq))
+            stack.push(ei.getNode());
+      }
+   }
+   return pos == this;
+}
+
+class DFSIterator : public Graph::GraphIterator
+{
+public:
+   DFSIterator(Graph *graph, const bool preorder)
+   {
+      unsigned int seq = graph->nextSequence();
+
+      nodes = new Graph::Node * [graph->getSize() + 1];
+      count = 0;
+      pos = 0;
+      nodes[graph->getSize()] = 0;
+
+      if (graph->getRoot()) {
+         graph->getRoot()->visit(seq);
+         search(graph->getRoot(), preorder, seq);
+      }
+   }
+
+   ~DFSIterator()
+   {
+      if (nodes)
+         delete[] nodes;
+   }
+
+   void search(Graph::Node *node, const bool preorder, const int sequence)
+   {
+      if (preorder)
+         nodes[count++] = node;
+
+      for (Graph::EdgeIterator ei = node->outgoing(); !ei.end(); ei.next())
+         if (ei.getNode()->visit(sequence))
+            search(ei.getNode(), preorder, sequence);
+
+      if (!preorder)
+         nodes[count++] = node;
+   }
+
+   virtual bool end() const { return pos >= count; }
+   virtual void next() { if (pos < count) ++pos; }
+   virtual void *get() const { return nodes[pos]; }
+
+   void reset() { pos = 0; }
+
+protected:
+   Graph::Node **nodes;
+   int count;
+   int pos;
+};
+
+Graph::GraphIterator *Graph::iteratorDFS(bool preorder)
+{
+   return new DFSIterator(this, preorder);
+}
+
+Graph::GraphIterator *Graph::safeIteratorDFS(bool preorder)
+{
+   return this->iteratorDFS(preorder);
+}
+
+class CFGIterator : public Graph::GraphIterator
+{
+public:
+   CFGIterator(Graph *graph)
+   {
+      nodes = new Graph::Node * [graph->getSize() + 1];
+      count = 0;
+      pos = 0;
+      nodes[graph->getSize()] = 0;
+
+      // TODO: argh, use graph->sequence instead of tag and just raise it by > 1
+      Iterator *iter;
+      for (iter = graph->iteratorDFS(); !iter->end(); iter->next())
+         reinterpret_cast<Graph::Node *>(iter->get())->tag = 0;
+      graph->putIterator(iter);
+
+      if (graph->getRoot())
+         search(graph->getRoot(), graph->nextSequence());
+   }
+
+   ~CFGIterator()
+   {
+      if (nodes)
+         delete[] nodes;
+   }
+
+   virtual void *get() const { return nodes[pos]; }
+   virtual bool end() const { return pos >= count; }
+   virtual void next() { if (pos < count) ++pos; }
+
+private:
+   void search(Graph::Node *node, const int sequence)
+   {
+      Stack bb, cross;
+
+      bb.push(node);
+
+      while (bb.getSize()) {
+         node = reinterpret_cast<Graph::Node *>(bb.pop().u.p);
+         assert(node);
+         if (!node->visit(sequence))
+            continue;
+         node->tag = 0;
+
+         for (Graph::EdgeIterator ei = node->outgoing(); !ei.end(); ei.next()) {
+            switch (ei.getType()) {
+            case Graph::Edge::TREE:
+            case Graph::Edge::FORWARD:
+            case Graph::Edge::DUMMY:
+               if (++(ei.getNode()->tag) == ei.getNode()->incidentCountFwd())
+                  bb.push(ei.getNode());
+               break;
+            case Graph::Edge::BACK:
+               continue;
+            case Graph::Edge::CROSS:
+               if (++(ei.getNode()->tag) == 1)
+                  cross.push(ei.getNode());
+               break;
+            default:
+               assert(!"unknown edge kind in CFG");
+               break;
+            }
+         }
+         nodes[count++] = node;
+
+         if (bb.getSize() == 0)
+            cross.moveTo(bb);
+      }
+   }
+
+private:
+   Graph::Node **nodes;
+   int count;
+   int pos;
+};
+
+Graph::GraphIterator *Graph::iteratorCFG()
+{
+   return new CFGIterator(this);
+}
+
+Graph::GraphIterator *Graph::safeIteratorCFG()
+{
+   return this->iteratorCFG();
+}
+
+void Graph::classifyEdges()
+{
+   DFSIterator *iter;
+   int seq;
+
+   for (iter = new DFSIterator(this, true); !iter->end(); iter->next()) {
+      Node *node = reinterpret_cast<Node *>(iter->get());
+      node->visit(0);
+      node->tag = 0;
+   }
+   putIterator(iter);
+
+   classifyDFS(root, (seq = 0));
+
+   sequence = seq;
+}
+
+void Graph::classifyDFS(Node *curr, int& seq)
+{
+   Graph::Edge *edge;
+   Graph::Node *node;
+
+   curr->visit(++seq);
+   curr->tag = 1;
+
+   for (edge = curr->out; edge; edge = edge->next[0]) {
+      node = edge->target;
+      if (edge->type == Edge::DUMMY)
+         continue;
+
+      if (node->getSequence() == 0) {
+         edge->type = Edge::TREE;
+         classifyDFS(node, seq);
+      } else
+      if (node->getSequence() > curr->getSequence()) {
+         edge->type = Edge::FORWARD;
+      } else {
+         edge->type = node->tag ? Edge::BACK : Edge::CROSS;
+      }
+   }
+
+   for (edge = curr->in; edge; edge = edge->next[1]) {
+      node = edge->origin;
+      if (edge->type == Edge::DUMMY)
+         continue;
+
+      if (node->getSequence() == 0) {
+         edge->type = Edge::TREE;
+         classifyDFS(node, seq);
+      } else
+      if (node->getSequence() > curr->getSequence()) {
+         edge->type = Edge::FORWARD;
+      } else {
+         edge->type = node->tag ? Edge::BACK : Edge::CROSS;
+      }
+   }
+
+   curr->tag = 0;
+}
+
+} // namespace nv50_ir
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_graph.h b/src/gallium/drivers/nv50/codegen/nv50_ir_graph.h
new file mode 100644
index 00000000000..6407ff98ab5
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_graph.h
@@ -0,0 +1,207 @@
+
+#ifndef __NV50_IR_GRAPH_H__
+#define __NV50_IR_GRAPH_H__
+
+#include "nv50_ir_util.h"
+
+namespace nv50_ir {
+
+#define ITER_NODE(x) reinterpret_cast<Graph::Node *>((x).get())
+#define ITER_EDGE(x) reinterpret_cast<Graph::Edge *>((x).get())
+
+// A connected graph.
+class Graph
+{
+public:
+   class Node;
+
+   class GraphIterator : public Iterator
+   {
+   public:
+      virtual ~GraphIterator() { };
+   };
+
+   class Edge
+   {
+   public:
+      enum Type
+      {
+         UNKNOWN,
+         TREE,
+         FORWARD,
+         BACK,
+         CROSS, // e.g. loop break
+         DUMMY
+      };
+
+      Edge(Node *dst, Node *src, Type kind);
+      ~Edge() { unlink(); }
+
+      inline Node *getOrigin() const { return origin; }
+      inline Node *getTarget() const { return target; }
+
+      inline Type getType() const { return type; }
+      const char *typeStr() const;
+
+   private:
+      Node *origin;
+      Node *target;
+
+      Type type;
+      Edge *next[2]; // next edge outgoing/incident from/to origin/target
+      Edge *prev[2];
+
+      void unlink();
+
+      friend class Graph;
+   };
+
+   class EdgeIterator : public Iterator
+   {
+   public:
+      EdgeIterator() : e(0), t(0), d(0) { }
+      EdgeIterator(Graph::Edge *first, int dir) : e(first), t(first), d(dir) { }
+
+      virtual void next() { e = (e->next[d] == t) ? 0 : e->next[d]; }
+      virtual bool end() const { return !e; }
+      virtual void *get() const { return e; }
+
+      inline Node *getNode() const { assert(e); return d ?
+                                                   e->origin : e->target; }
+      inline Edge *getEdge() const { return e; }
+      inline Edge::Type getType() { return e ? e->getType() : Edge::UNKNOWN; }
+
+   private:
+      Graph::Edge *e;
+      Graph::Edge *t;
+      int d;
+   };
+
+   class Node
+   {
+   public:
+      Node(void *);
+      ~Node() { cut(); }
+
+      void attach(Node *, Edge::Type);
+      bool detach(Node *);
+      void cut();
+
+      inline EdgeIterator outgoing() const;
+      inline EdgeIterator incident() const;
+
+      inline Node *parent() const; // returns NULL if count(incident edges) != 1
+
+      bool reachableBy(Node *node, Node *term);
+
+      inline bool visit(int);
+      inline int  getSequence() const;
+
+      inline int incidentCountFwd() const; // count of incident non-back edges
+      inline int incidentCount() const { return inCount; }
+      inline int outgoingCount() const { return outCount; }
+
+      Graph *getGraph() const { return graph; }
+
+      void *data;
+
+   private:
+      Edge *in;
+      Edge *out;
+      Graph *graph;
+
+      int visited;
+
+      int16_t inCount;
+      int16_t outCount;
+   public:
+      int tag; // for temporary use
+
+      friend class Graph;
+   };
+
+public:
+   Graph();
+   ~Graph(); // does *not* free the nodes (make it an option ?)
+
+   inline Node *getRoot() const { return root; }
+
+   inline unsigned int getSize() const { return size; }
+
+   inline int nextSequence();
+
+   void insert(Node *node); // attach to or set as root
+
+   GraphIterator *iteratorDFS(bool preorder = true);
+   GraphIterator *iteratorCFG();
+
+   // safe iterators are unaffected by changes to the *edges* of the graph
+   GraphIterator *safeIteratorDFS(bool preorder = true);
+   GraphIterator *safeIteratorCFG();
+
+   inline void putIterator(Iterator *); // should be GraphIterator *
+
+   void classifyEdges();
+
+private:
+   void classifyDFS(Node *, int&);
+
+private:
+   Node *root;
+   unsigned int size;
+   int sequence;
+};
+
+int Graph::nextSequence()
+{
+   return ++sequence;
+}
+
+Graph::Node *Graph::Node::parent() const
+{
+   if (inCount != 1)
+      return NULL;
+   assert(in);
+   return in->origin;
+}
+
+bool Graph::Node::visit(int v)
+{
+   if (visited == v)
+      return false;
+   visited = v;
+   return true;
+}
+
+int Graph::Node::getSequence() const
+{
+   return visited;
+}
+
+void Graph::putIterator(Iterator *iter)
+{
+   delete reinterpret_cast<GraphIterator *>(iter);
+}
+
+Graph::EdgeIterator Graph::Node::outgoing() const
+{
+   return EdgeIterator(out, 0);
+}
+
+Graph::EdgeIterator Graph::Node::incident() const
+{
+   return EdgeIterator(in, 1);
+}
+
+int Graph::Node::incidentCountFwd() const
+{
+   int n = 0;
+   for (EdgeIterator ei = incident(); !ei.end(); ei.next())
+      if (ei.getType() != Edge::BACK)
+         ++n;
+   return n;
+}
+
+} // namespace nv50_ir
+
+#endif // __NV50_IR_GRAPH_H__
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_inlines.h b/src/gallium/drivers/nv50/codegen/nv50_ir_inlines.h
new file mode 100644
index 00000000000..8730e953482
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_inlines.h
@@ -0,0 +1,328 @@
+
+#ifndef __NV50_IR_INLINES_H__
+#define __NV50_IR_INLINES_H__
+
+static inline CondCode reverseCondCode(CondCode cc)
+{
+   static const uint8_t ccRev[8] = { 0, 4, 2, 6, 1, 5, 3, 7 };
+
+   return static_cast<CondCode>(ccRev[cc & 7] | (cc & ~7));
+}
+
+static inline CondCode inverseCondCode(CondCode cc)
+{
+   return static_cast<CondCode>(cc ^ 7);
+}
+
+static inline bool isMemoryFile(DataFile f)
+{
+   return (f >= FILE_MEMORY_CONST && f <= FILE_MEMORY_LOCAL);
+}
+
+static inline bool isTextureOp(operation op)
+{
+   return (op >= OP_TEX && op <= OP_TEXCSAA);
+}
+
+static inline unsigned int typeSizeof(DataType ty)
+{
+   switch (ty) {
+   case TYPE_U8:
+   case TYPE_S8:
+      return 1;
+   case TYPE_F16:
+   case TYPE_U16:
+   case TYPE_S16:
+      return 2;
+   case TYPE_F32:
+   case TYPE_U32:
+   case TYPE_S32:
+      return 4;
+   case TYPE_F64:
+   case TYPE_U64:
+   case TYPE_S64:
+      return 8;
+   case TYPE_B96:
+      return 12;
+   case TYPE_B128:
+      return 16;
+   default:
+      return 0;
+   }
+}
+
+static inline DataType typeOfSize(unsigned int size,
+                                  bool flt = false, bool sgn = false)
+{
+   switch (size) {
+   case 1: return sgn ? TYPE_S8 : TYPE_U8;
+   case 2: return flt ? TYPE_F16 : (sgn ? TYPE_S16 : TYPE_U16);
+   case 8: return flt ? TYPE_F64 : (sgn ? TYPE_S64 : TYPE_U64);
+   case 12: return TYPE_B96;
+   case 16: return TYPE_B128;
+   case 4:
+   default:
+      return flt ? TYPE_F32 : (sgn ? TYPE_S32 : TYPE_U32);
+   }
+}
+
+static inline bool isFloatType(DataType ty)
+{
+   return (ty >= TYPE_F16 && ty <= TYPE_F64);
+}
+
+static inline bool isSignedIntType(DataType ty)
+{
+   return (ty == TYPE_S8 || ty == TYPE_S16 || ty == TYPE_S32);
+}
+
+static inline bool isSignedType(DataType ty)
+{
+   switch (ty) {
+   case TYPE_NONE:
+   case TYPE_U8:
+   case TYPE_U16:
+   case TYPE_U32:
+   case TYPE_B96:
+   case TYPE_B128:
+      return false;
+   default:
+      return true;
+   }
+}
+
+const ValueRef *ValueRef::getIndirect(int dim) const
+{
+   return isIndirect(dim) ? &insn->src[indirect[dim]] : NULL;
+}
+
+DataFile ValueRef::getFile() const
+{
+   return value ? value->reg.file : FILE_NULL;
+}
+
+unsigned int ValueRef::getSize() const
+{
+   return value ? value->reg.size : 0;
+}
+
+Value *ValueRef::rep() const
+{
+   assert(value);
+   return value->join;
+}
+
+Value *ValueDef::rep() const
+{
+   assert(value);
+   return value->join;
+}
+
+DataFile ValueDef::getFile() const
+{
+   return value ? value->reg.file : FILE_NULL;
+}
+
+unsigned int ValueDef::getSize() const
+{
+   return value ? value->reg.size : 0;
+}
+
+void ValueDef::setSSA(LValue *lval)
+{
+   Value *save = value;
+
+   this->set(NULL);
+   prev = reinterpret_cast<ValueDef *>(save);
+   value = lval;
+   lval->defs = this;
+}
+
+void ValueDef::restoreDefList()
+{
+   if (next == this)
+      prev = this;
+}
+
+const LValue *ValueDef::preSSA() const
+{
+   return reinterpret_cast<LValue *>(prev);
+}
+
+Instruction *Value::getInsn() const
+{
+   assert(!defs || getUniqueInsn());
+   return defs ? defs->getInsn() : NULL;
+}
+
+Instruction *Value::getUniqueInsn() const
+{
+   if (defs) {
+      if (join != this) {
+         ValueDef::Iterator it = defs->iterator();
+         while (!it.end() && it.get()->get() != this)
+            it.next();
+         assert(it.get()->get() == this);
+         return it.get()->getInsn();
+      }
+
+      // after regalloc, the definitions of coalesced values are linked
+      if (reg.data.id < 0) {
+         ValueDef::Iterator it = defs->iterator();
+         int nDef;
+         for (nDef = 0; !it.end() && nDef < 2; it.next())
+            if (it.get()->get() == this) // don't count joined values
+               ++nDef;
+         if (nDef > 1)
+            WARN("value %%%i not uniquely defined\n", id); // return NULL ?
+      }
+
+      assert(defs->get() == this);
+      return defs->getInsn();
+   }
+   return NULL;
+}
+
+Value *Instruction::getIndirect(int s, int dim) const
+{
+   return src[s].isIndirect(dim) ? getSrc(src[s].indirect[dim]) : NULL;
+}
+
+Value *Instruction::getPredicate() const
+{
+   return (predSrc >= 0) ? getSrc(predSrc) : NULL;
+}
+
+Value *TexInstruction::getIndirectR() const
+{
+   return tex.rIndirectSrc >= 0 ? getSrc(tex.rIndirectSrc) : NULL;
+}
+
+Value *TexInstruction::getIndirectS() const
+{
+   return tex.rIndirectSrc >= 0 ? getSrc(tex.rIndirectSrc) : NULL;
+}
+
+CmpInstruction *Instruction::asCmp()
+{
+   if (op >= OP_SET_AND && op <= OP_SLCT && op != OP_SELP)
+      return static_cast<CmpInstruction *>(this);
+   return NULL;
+}
+
+const CmpInstruction *Instruction::asCmp() const
+{
+   if (op >= OP_SET_AND && op <= OP_SLCT && op != OP_SELP)
+      return static_cast<const CmpInstruction *>(this);
+   return NULL;
+}
+
+FlowInstruction *Instruction::asFlow()
+{
+   if (op >= OP_BRA && op <= OP_JOIN)
+      return static_cast<FlowInstruction *>(this);
+   return NULL;
+}
+
+const FlowInstruction *Instruction::asFlow() const
+{
+   if (op >= OP_BRA && op <= OP_JOINAT)
+      return static_cast<const FlowInstruction *>(this);
+   return NULL;
+}
+
+TexInstruction *Instruction::asTex()
+{
+   if (op >= OP_TEX && op <= OP_TEXCSAA)
+      return static_cast<TexInstruction *>(this);
+   return NULL;
+}
+
+const TexInstruction *Instruction::asTex() const
+{
+   if (op >= OP_TEX && op <= OP_TEXCSAA)
+      return static_cast<const TexInstruction *>(this);
+   return NULL;
+}
+
+// XXX: use a virtual function so we're really really safe ?
+LValue *Value::asLValue()
+{
+   if (reg.file >= FILE_GPR && reg.file <= FILE_ADDRESS)
+      return static_cast<LValue *>(this);
+   return NULL;
+}
+
+Symbol *Value::asSym()
+{
+   if (reg.file >= FILE_MEMORY_CONST)
+      return static_cast<Symbol *>(this);
+   return NULL;
+}
+
+const Symbol *Value::asSym() const
+{
+   if (reg.file >= FILE_MEMORY_CONST)
+      return static_cast<const Symbol *>(this);
+   return NULL;
+}
+
+void Symbol::setOffset(int32_t offset)
+{
+   reg.data.offset = offset;
+}
+
+void Symbol::setAddress(Symbol *base, int32_t offset)
+{
+   baseSym = base;
+   reg.data.offset = offset;
+}
+
+void Symbol::setSV(SVSemantic sv, uint32_t index)
+{
+   reg.data.sv.sv = sv;
+   reg.data.sv.index = index;
+}
+
+ImmediateValue *Value::asImm()
+{
+   if (reg.file == FILE_IMMEDIATE)
+      return static_cast<ImmediateValue *>(this);
+   return NULL;
+}
+
+const ImmediateValue *Value::asImm() const
+{
+   if (reg.file == FILE_IMMEDIATE)
+      return static_cast<const ImmediateValue *>(this);
+   return NULL;
+}
+
+Value *Value::get(Iterator &it)
+{
+   return reinterpret_cast<Value *>(it.get());
+}
+
+bool BasicBlock::reachableBy(BasicBlock *by, BasicBlock *term)
+{
+   return cfg.reachableBy(&by->cfg, &term->cfg);
+}
+
+BasicBlock *BasicBlock::get(Iterator &iter)
+{
+   return reinterpret_cast<BasicBlock *>(iter.get());
+}
+
+BasicBlock *BasicBlock::get(Graph::Node *node)
+{
+   assert(node);
+   return reinterpret_cast<BasicBlock *>(node->data);
+}
+
+LValue *Function::getLValue(int id)
+{
+   assert((unsigned int)id < (unsigned int)allLValues.getSize());
+   return reinterpret_cast<LValue *>(allLValues.get(id));
+}
+
+#endif // __NV50_IR_INLINES_H__
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_peephole.cpp b/src/gallium/drivers/nv50/codegen/nv50_ir_peephole.cpp
new file mode 100644
index 00000000000..bd331ea8f03
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_peephole.cpp
@@ -0,0 +1,2192 @@
+
+#include "nv50_ir.h"
+#include "nv50_ir_target.h"
+#include "nv50_ir_build_util.h"
+
+extern "C" {
+#include "util/u_math.h"
+}
+
+namespace nv50_ir {
+
+bool
+Instruction::isNop() const
+{
+   if (op == OP_CONSTRAINT || op == OP_PHI)
+      return true;
+   if (terminator || join) // XXX: should terminator imply flow ?
+      return false;
+   if (!fixed && op == OP_NOP)
+      return true;
+
+   if (def[0].exists() && def[0].rep()->reg.data.id < 0) {
+      for (int d = 1; defExists(d); ++d)
+         if (def[d].rep()->reg.data.id >= 0)
+            WARN("part of vector result is unused !\n");
+      return true;
+   }
+
+   if (op == OP_MOV || op == OP_UNION) {
+      if (!def[0].rep()->equals(getSrc(0)))
+         return false;
+      if (op == OP_UNION)
+         if (!def[0].rep()->equals(getSrc(1)))
+            return false;
+      return true;
+   }
+
+   return false;
+}
+
+bool Instruction::isDead() const
+{
+   if (op == OP_STORE ||
+       op == OP_EXPORT)
+      return false;
+
+   for (int d = 0; defExists(d); ++d)
+      if (getDef(d)->refCount() || getDef(d)->reg.data.id >= 0)
+         return false;
+
+   if (terminator || asFlow())
+      return false;
+   if (fixed)
+      return false;
+
+   return true;
+};
+
+// =============================================================================
+
+class CopyPropagation : public Pass
+{
+private:
+   virtual bool visit(BasicBlock *);
+};
+
+// Propagate all MOVs forward to make subsequent optimization easier, except if
+// the sources stem from a phi, in which case we don't want to mess up potential
+// swaps $rX <-> $rY, i.e. do not create live range overlaps of phi src and def.
+bool
+CopyPropagation::visit(BasicBlock *bb)
+{
+   Instruction *mov, *si, *next;
+
+   for (mov = bb->getEntry(); mov; mov = next) {
+      next = mov->next;
+      if (mov->op != OP_MOV || mov->fixed || !mov->getSrc(0)->asLValue())
+         continue;
+      si = mov->getSrc(0)->getInsn();
+      if (mov->getDef(0)->reg.data.id < 0 && si && si->op != OP_PHI) {
+         // propagate
+         mov->def[0].replace(mov->getSrc(0), false);
+         delete_Instruction(prog, mov);
+      }
+   }
+   return true;
+}
+
+// =============================================================================
+
+class LoadPropagation : public Pass
+{
+private:
+   virtual bool visit(BasicBlock *);
+
+   void checkSwapSrc01(Instruction *);
+
+   bool isCSpaceLoad(Instruction *);
+   bool isImmd32Load(Instruction *);
+};
+
+bool
+LoadPropagation::isCSpaceLoad(Instruction *ld)
+{
+   return ld && ld->op == OP_LOAD && ld->src[0].getFile() == FILE_MEMORY_CONST;
+}
+
+bool
+LoadPropagation::isImmd32Load(Instruction *ld)
+{
+   if (!ld || (ld->op != OP_MOV) || (typeSizeof(ld->dType) != 4))
+      return false;
+   return ld->src[0].getFile() == FILE_IMMEDIATE;
+}
+
+void
+LoadPropagation::checkSwapSrc01(Instruction *insn)
+{
+   if (!prog->getTarget()->getOpInfo(insn).commutative)
+      if (insn->op != OP_SET && insn->op != OP_SLCT)
+         return;
+   if (insn->src[1].getFile() != FILE_GPR)
+      return;
+
+   Instruction *i0 = insn->getSrc(0)->getInsn();
+   Instruction *i1 = insn->getSrc(1)->getInsn();
+
+   if (isCSpaceLoad(i0)) {
+      if (!isCSpaceLoad(i1))
+         insn->swapSources(0, 1);
+      else
+         return;
+   } else
+   if (isImmd32Load(i0)) {
+      if (!isCSpaceLoad(i1) && !isImmd32Load(i1))
+         insn->swapSources(0, 1);
+      else
+         return;
+   } else {
+      return;
+   }
+
+   if (insn->op == OP_SET)
+      insn->asCmp()->setCond = reverseCondCode(insn->asCmp()->setCond);
+   else
+   if (insn->op == OP_SLCT)
+      insn->asCmp()->setCond = inverseCondCode(insn->asCmp()->setCond);
+}
+
+bool
+LoadPropagation::visit(BasicBlock *bb)
+{
+   const Target *targ = prog->getTarget();
+   Instruction *next;
+
+   for (Instruction *i = bb->getEntry(); i; i = next) {
+      next = i->next;
+
+      if (i->srcExists(1))
+         checkSwapSrc01(i);
+
+      for (int s = 0; i->srcExists(s); ++s) {
+         Instruction *ld = i->getSrc(s)->getInsn();
+
+         if (!ld || ld->fixed || (ld->op != OP_LOAD && ld->op != OP_MOV))
+            continue;
+         if (!targ->insnCanLoad(i, s, ld))
+            continue;
+
+         // propagate !
+         i->setSrc(s, ld->getSrc(0));
+         if (ld->src[0].isIndirect(0))
+            i->setIndirect(s, 0, ld->getIndirect(0, 0));
+
+         if (ld->getDef(0)->refCount() == 0)
+            delete_Instruction(prog, ld);
+      }
+   }
+   return true;
+}
+
+// =============================================================================
+
+// Evaluate constant expressions.
+class ConstantFolding : public Pass
+{
+public:
+   bool foldAll(Program *);
+
+private:
+   virtual bool visit(BasicBlock *);
+
+   void expr(Instruction *, ImmediateValue *, ImmediateValue *);
+   void opnd(Instruction *, ImmediateValue *, int s);
+
+   void unary(Instruction *, const ImmediateValue&);
+
+   // TGSI 'true' is converted to -1 by F2I(NEG(SET)), track back to SET
+   CmpInstruction *findOriginForTestWithZero(Value *);
+
+   unsigned int foldCount;
+
+   BuildUtil bld;
+};
+
+// TODO: remember generated immediates and only revisit these
+bool
+ConstantFolding::foldAll(Program *prog)
+{
+   unsigned int iterCount = 0;
+   do {
+      foldCount = 0;
+      if (!run(prog))
+         return false;
+   } while (foldCount && ++iterCount < 2);
+   return true;
+}
+
+bool
+ConstantFolding::visit(BasicBlock *bb)
+{
+   Instruction *i, *next;
+
+   for (i = bb->getEntry(); i; i = next) {
+      next = i->next;
+      if (i->op == OP_MOV) // continue early, MOV appears frequently
+         continue;
+
+      ImmediateValue *src0 = i->src[0].getImmediate();
+      ImmediateValue *src1 = i->src[1].getImmediate();
+
+      if (src0 && src1)
+         expr(i, src0, src1);
+      else
+      if (src0)
+         opnd(i, src0, 0);
+      else
+      if (src1)
+         opnd(i, src1, 1);
+   }
+   return true;
+}
+
+CmpInstruction *
+ConstantFolding::findOriginForTestWithZero(Value *value)
+{
+   if (!value)
+      return NULL;
+   Instruction *insn = value->getInsn();
+
+   while (insn && insn->op != OP_SET) {
+      Instruction *next = NULL;
+      switch (insn->op) {
+      case OP_NEG:
+      case OP_ABS:
+      case OP_CVT:
+         next = insn->getSrc(0)->getInsn();
+         if (insn->sType != next->dType)
+            return NULL;
+         break;
+      case OP_MOV:
+         next = insn->getSrc(0)->getInsn();
+         break;
+      default:
+         return NULL;
+      }
+      insn = next;
+   }
+   return insn ? insn->asCmp() : NULL;
+}
+
+void
+Modifier::applyTo(ImmediateValue& imm) const
+{
+   switch (imm.reg.type) {
+   case TYPE_F32:
+      if (bits & NV50_IR_MOD_ABS)
+         imm.reg.data.f32 = fabsf(imm.reg.data.f32);
+      if (bits & NV50_IR_MOD_NEG)
+         imm.reg.data.f32 = -imm.reg.data.f32;
+      if (bits & NV50_IR_MOD_SAT) {
+         if (imm.reg.data.f32 < 0.0f)
+            imm.reg.data.f32 = 0.0f;
+         else
+         if (imm.reg.data.f32 > 1.0f)
+            imm.reg.data.f32 = 1.0f;
+      }
+      assert(!(bits & NV50_IR_MOD_NOT));
+      break;
+
+   case TYPE_S8: // NOTE: will be extended
+   case TYPE_S16:
+   case TYPE_S32:
+   case TYPE_U8: // NOTE: treated as signed
+   case TYPE_U16:
+   case TYPE_U32:
+      if (bits & NV50_IR_MOD_ABS)
+         imm.reg.data.s32 = (imm.reg.data.s32 >= 0) ?
+            imm.reg.data.s32 : -imm.reg.data.s32;
+      if (bits & NV50_IR_MOD_NEG)
+         imm.reg.data.s32 = -imm.reg.data.s32;
+      if (bits & NV50_IR_MOD_NOT)
+         imm.reg.data.s32 = ~imm.reg.data.s32;
+      break;
+
+   case TYPE_F64:
+      if (bits & NV50_IR_MOD_ABS)
+         imm.reg.data.f64 = fabs(imm.reg.data.f64);
+      if (bits & NV50_IR_MOD_NEG)
+         imm.reg.data.f64 = -imm.reg.data.f64;
+      if (bits & NV50_IR_MOD_SAT) {
+         if (imm.reg.data.f64 < 0.0)
+            imm.reg.data.f64 = 0.0;
+         else
+         if (imm.reg.data.f64 > 1.0)
+            imm.reg.data.f64 = 1.0;
+      }
+      assert(!(bits & NV50_IR_MOD_NOT));
+      break;
+
+   default:
+      assert(!"invalid/unhandled type");
+      imm.reg.data.u64 = 0;
+      break;
+   }
+}
+
+operation
+Modifier::getOp() const
+{
+   switch (bits) {
+   case NV50_IR_MOD_ABS: return OP_ABS;
+   case NV50_IR_MOD_NEG: return OP_NEG;
+   case NV50_IR_MOD_SAT: return OP_SAT;
+   case NV50_IR_MOD_NOT: return OP_NOT;
+   case 0:
+      return OP_MOV;
+   default:
+      return OP_CVT;
+   }
+}
+
+void
+ConstantFolding::expr(Instruction *i,
+                      ImmediateValue *src0, ImmediateValue *src1)
+{
+   ImmediateValue imm0(src0, i->sType);
+   ImmediateValue imm1(src1, i->sType);
+   struct Storage res;
+   struct Storage *const a = &imm0.reg, *const b = &imm1.reg;
+
+   i->src[0].mod.applyTo(imm0);
+   i->src[1].mod.applyTo(imm1);
+
+   switch (i->op) {
+   case OP_MAD:
+   case OP_FMA:
+   case OP_MUL:
+      if (i->dnz && i->dType == TYPE_F32) {
+         if (!isfinite(a->data.f32))
+            a->data.f32 = 0.0f;
+         if (!isfinite(b->data.f32))
+            b->data.f32 = 0.0f;
+      }
+      switch (i->dType) {
+      case TYPE_F32: res.data.f32 = a->data.f32 * b->data.f32; break;
+      case TYPE_F64: res.data.f64 = a->data.f64 * b->data.f64; break;
+      case TYPE_S32:
+      case TYPE_U32: res.data.u32 = a->data.u32 * b->data.u32; break;
+      default:
+         return;
+      }
+      break;
+   case OP_DIV:
+      if (b->data.u32 == 0)
+         break;
+      switch (i->dType) {
+      case TYPE_F32: res.data.f32 = a->data.f32 / b->data.f32; break;
+      case TYPE_F64: res.data.f64 = a->data.f64 / b->data.f64; break;
+      case TYPE_S32: res.data.s32 = a->data.s32 / b->data.s32; break;
+      case TYPE_U32: res.data.u32 = a->data.u32 / b->data.u32; break;
+      default:
+         return;
+      }
+      break;
+   case OP_ADD:
+      switch (i->dType) {
+      case TYPE_F32: res.data.f32 = a->data.f32 + b->data.f32; break;
+      case TYPE_F64: res.data.f64 = a->data.f64 + b->data.f64; break;
+      case TYPE_S32:
+      case TYPE_U32: res.data.u32 = a->data.u32 + b->data.u32; break;
+      default:
+         return;
+      }
+      break;
+   case OP_POW:
+      switch (i->dType) {
+      case TYPE_F32: res.data.f32 = pow(a->data.f32, b->data.f32); break;
+      case TYPE_F64: res.data.f64 = pow(a->data.f64, b->data.f64); break;
+      default:
+         return;
+      }
+      break;
+   case OP_MAX:
+      switch (i->dType) {
+      case TYPE_F32: res.data.f32 = MAX2(a->data.f32, b->data.f32); break;
+      case TYPE_F64: res.data.f64 = MAX2(a->data.f64, b->data.f64); break;
+      case TYPE_S32: res.data.s32 = MAX2(a->data.s32, b->data.s32); break;
+      case TYPE_U32: res.data.u32 = MAX2(a->data.u32, b->data.u32); break;
+      default:
+         return;
+      }
+      break;
+   case OP_MIN:
+      switch (i->dType) {
+      case TYPE_F32: res.data.f32 = MIN2(a->data.f32, b->data.f32); break;
+      case TYPE_F64: res.data.f64 = MIN2(a->data.f64, b->data.f64); break;
+      case TYPE_S32: res.data.s32 = MIN2(a->data.s32, b->data.s32); break;
+      case TYPE_U32: res.data.u32 = MIN2(a->data.u32, b->data.u32); break;
+      default:
+         return;
+      }
+      break;
+   case OP_AND:
+      res.data.u64 = a->data.u64 & b->data.u64;
+      break;
+   case OP_OR:
+      res.data.u64 = a->data.u64 | b->data.u64;
+      break;
+   case OP_XOR:
+      res.data.u64 = a->data.u64 ^ b->data.u64;
+      break;
+   case OP_SHL:
+      res.data.u32 = a->data.u32 << b->data.u32;
+      break;
+   case OP_SHR:
+      switch (i->dType) {
+      case TYPE_S32: res.data.s32 = a->data.s32 >> b->data.u32; break;
+      case TYPE_U32: res.data.u32 = a->data.u32 >> b->data.u32; break;
+      default:
+         return;
+      }
+      break;
+   case OP_SLCT:
+      if (a->data.u32 != b->data.u32)
+         return;
+      res.data.u32 = a->data.u32;
+      break;
+   default:
+      return;
+   }
+   ++foldCount;
+
+   i->src[0].mod = Modifier(0);
+   i->src[1].mod = Modifier(0);
+
+   i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res.data.u32));
+   i->setSrc(1, NULL);
+
+   i->getSrc(0)->reg.data = res.data;
+
+   if (i->op == OP_MAD || i->op == OP_FMA) {
+      i->op = OP_ADD;
+
+      i->setSrc(1, i->getSrc(0));
+      i->setSrc(0, i->getSrc(2));
+      i->setSrc(2, NULL);
+
+      i->src[1].mod = i->src[2].mod;
+
+      src0 = i->src[0].getImmediate();
+      if (src0)
+         expr(i, src0, i->getSrc(1)->asImm());
+   } else {
+      i->op = OP_MOV;
+   }
+}
+
+void
+ConstantFolding::unary(Instruction *i, const ImmediateValue &imm)
+{
+   Storage res;
+
+   if (i->dType != TYPE_F32)
+      return;
+   switch (i->op) {
+   case OP_NEG: res.data.f32 = -imm.reg.data.f32; break;
+   case OP_ABS: res.data.f32 = fabsf(imm.reg.data.f32); break;
+   case OP_RCP: res.data.f32 = 1.0f / imm.reg.data.f32; break;
+   case OP_RSQ: res.data.f32 = 1.0f / sqrtf(imm.reg.data.f32); break;
+   case OP_LG2: res.data.f32 = log2f(imm.reg.data.f32); break;
+   case OP_EX2: res.data.f32 = exp2f(imm.reg.data.f32); break;
+   case OP_SIN: res.data.f32 = sinf(imm.reg.data.f32); break;
+   case OP_COS: res.data.f32 = cosf(imm.reg.data.f32); break;
+   case OP_SQRT: res.data.f32 = sqrtf(imm.reg.data.f32); break;
+   case OP_PRESIN:
+   case OP_PREEX2:
+      // these should be handled in subsequent OP_SIN/COS/EX2
+      res.data.f32 = imm.reg.data.f32;
+      break;
+   default:
+      return;
+   }
+   i->op = OP_MOV;
+   i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res.data.f32));
+   i->src[0].mod = Modifier(0);
+}
+
+void
+ConstantFolding::opnd(Instruction *i, ImmediateValue *src, int s)
+{
+   const int t = !s;
+   const operation op = i->op;
+
+   ImmediateValue imm(src, i->sType);
+
+   i->src[s].mod.applyTo(imm);
+
+   switch (i->op) {
+   case OP_MUL:
+      if (i->dType == TYPE_F32 && i->getSrc(t)->refCount() == 1) {
+         Instruction *si = i->getSrc(t)->getUniqueInsn();
+
+         if (si && si->op == OP_MUL) {
+            float f = imm.reg.data.f32;
+
+            if (si->src[1].getImmediate()) {
+               f *= si->src[1].getImmediate()->reg.data.f32;
+               si->setSrc(1, new_ImmediateValue(prog, f));
+               i->def[0].replace(i->getSrc(t), false);
+               break;
+            } else {
+               int fac;
+               if (f == 0.125f) fac = -3;
+               else
+               if (f == 0.250f) fac = -2;
+               else
+               if (f == 0.500f) fac = -1;
+               else
+               if (f == 2.000f) fac = +1;
+               else
+               if (f == 4.000f) fac = +2;
+               else
+               if (f == 8.000f) fac = +3;
+               else
+                  fac = 0;
+               if (fac) {
+                  // FIXME: allowed & modifier
+                  si->postFactor = fac;
+                  i->def[0].replace(i->getSrc(t), false);
+                  break;
+               }
+            }
+         }
+      }
+      if (imm.isInteger(0)) {
+         i->op = OP_MOV;
+         i->setSrc(0, i->getSrc(s));
+         i->setSrc(1, NULL);
+      } else
+      if (imm.isInteger(1) || imm.isInteger(-1)) {
+         if (imm.isNegative())
+            i->src[t].mod = i->src[t].mod ^ Modifier(NV50_IR_MOD_NEG);
+         i->op = i->src[t].mod.getOp();
+         if (s == 0) {
+            i->setSrc(0, i->getSrc(1));
+            i->src[0].mod = i->src[1].mod;
+            i->src[1].mod = 0;
+         }
+         if (i->op != OP_CVT)
+            i->src[0].mod = 0;
+         i->setSrc(1, NULL);
+      } else
+      if (imm.isInteger(2) || imm.isInteger(-2)) {
+         if (imm.isNegative())
+            i->src[t].mod = i->src[t].mod ^ Modifier(NV50_IR_MOD_NEG);
+         i->op = OP_ADD;
+         i->setSrc(s, i->getSrc(t));
+         i->src[s].mod = i->src[t].mod;
+      } else
+      if (!isFloatType(i->sType) && !imm.isNegative() && imm.isPow2()) {
+         i->op = OP_SHL;
+         imm.applyLog2();
+         i->setSrc(1, new_ImmediateValue(prog, imm.reg.data.u32));
+      }
+      break;
+   case OP_ADD:
+      if (imm.isInteger(0)) {
+         if (s == 0) {
+            i->setSrc(0, i->getSrc(1));
+            i->src[0].mod = i->src[1].mod;
+         }
+         i->setSrc(1, NULL);
+         i->op = i->src[0].mod.getOp();
+         if (i->op != OP_CVT)
+            i->src[0].mod = Modifier(0);
+      }
+      break;
+
+   case OP_DIV:
+      if (s != 1 || (i->dType != TYPE_S32 && i->dType != TYPE_U32))
+         break;
+      bld.setPosition(i, false);
+      if (imm.reg.data.u32 == 0) {
+         break;
+      } else
+      if (imm.reg.data.u32 == 1) {
+         i->op = OP_MOV;
+         i->setSrc(1, NULL);
+      } else
+      if (i->dType == TYPE_U32 && imm.isPow2()) {
+         i->op = OP_SHL;
+         i->setSrc(1, bld.mkImm(util_logbase2(imm.reg.data.u32)));
+      } else
+      if (i->dType == TYPE_U32) {
+         Instruction *mul;
+         Value *tA, *tB;
+         const uint32_t d = imm.reg.data.u32;
+         uint32_t m;
+         int r, s;
+         uint32_t l = util_logbase2(d);
+         if (((uint32_t)1 << l) < d)
+            ++l;
+         m = (((uint64_t)1 << 32) * (((uint64_t)1 << l) - d)) / d + 1;
+         r = l ? 1 : 0;
+         s = l ? (l - 1) : 0;
+
+         tA = bld.getSSA();
+         tB = bld.getSSA();
+         mul = bld.mkOp2(OP_MUL, TYPE_U32, tA, i->getSrc(0),
+                         bld.loadImm(NULL, m));
+         mul->subOp = NV50_IR_SUBOP_MUL_HIGH;
+         bld.mkOp2(OP_SUB, TYPE_U32, tB, i->getSrc(0), tA);
+         tA = bld.getSSA();
+         if (r)
+            bld.mkOp2(OP_SHR, TYPE_U32, tA, tB, bld.mkImm(r));
+         else
+            tA = tB;
+         tB = s ? bld.getSSA() : i->getDef(0);
+         bld.mkOp2(OP_ADD, TYPE_U32, tB, mul->getDef(0), tA);
+         if (s)
+            bld.mkOp2(OP_SHR, TYPE_U32, i->getDef(0), tB, bld.mkImm(s));
+
+         delete_Instruction(prog, i);
+      } else
+      if (imm.reg.data.s32 == -1) {
+         i->op = OP_NEG;
+         i->setSrc(1, NULL);
+      } else {
+         LValue *tA, *tB;
+         LValue *tD;
+         const int32_t d = imm.reg.data.s32;
+         int32_t m;
+         int32_t l = util_logbase2(static_cast<unsigned>(abs(d)));
+         if ((1 << l) < abs(d))
+            ++l;
+         if (!l)
+            l = 1;
+         m = ((uint64_t)1 << (32 + l - 1)) / abs(d) + 1 - ((uint64_t)1 << 32);
+
+         tA = bld.getSSA();
+         tB = bld.getSSA();
+         bld.mkOp3(OP_MAD, TYPE_S32, tA, i->getSrc(0), bld.loadImm(NULL, m),
+                   i->getSrc(0))->subOp = NV50_IR_SUBOP_MUL_HIGH;
+         if (l > 1)
+            bld.mkOp2(OP_SHR, TYPE_S32, tB, tA, bld.mkImm(l - 1));
+         else
+            tB = tA;
+         tA = bld.getSSA();
+         bld.mkCmp(OP_SET, CC_LT, TYPE_S32, tA, i->getSrc(0), bld.mkImm(0));
+         tD = (d < 0) ? bld.getSSA() : i->getDef(0)->asLValue();
+         bld.mkOp2(OP_SUB, TYPE_U32, tD, tB, tA);
+         if (d < 0)
+            bld.mkOp1(OP_NEG, TYPE_S32, i->getDef(0), tB);
+
+         delete_Instruction(prog, i);
+      }
+      break;
+
+   case OP_SET: // TODO: SET_AND,OR,XOR
+   {
+      CmpInstruction *si = findOriginForTestWithZero(i->getSrc(t));
+      CondCode cc, ccZ;
+      if (i->src[t].mod != Modifier(0))
+         return;
+      if (imm.reg.data.u32 != 0 || !si || si->op != OP_SET)
+         return;
+      cc = si->setCond;
+      ccZ = (CondCode)((unsigned int)i->asCmp()->setCond & ~CC_U);
+      if (s == 0)
+         ccZ = reverseCondCode(ccZ);
+      switch (ccZ) {
+      case CC_LT: cc = CC_FL; break;
+      case CC_GE: cc = CC_TR; break;
+      case CC_EQ: cc = inverseCondCode(cc); break;
+      case CC_LE: cc = inverseCondCode(cc); break;
+      case CC_GT: break;
+      case CC_NE: break;
+      default:
+         return;
+      }
+      i->asCmp()->setCond = cc;
+      i->setSrc(0, si->src[0]);
+      i->setSrc(1, si->src[1]);
+      i->sType = si->sType;
+   }
+      break;
+
+   case OP_SHL:
+   {
+      if (s != 1 || i->src[0].mod != Modifier(0))
+         break;
+      // try to concatenate shifts
+      Instruction *si = i->getSrc(0)->getInsn();
+      if (!si ||
+          si->op != OP_SHL || si->src[1].mod != Modifier(0))
+         break;
+      ImmediateValue *siImm = si->src[1].getImmediate();
+      if (siImm) {
+         bld.setPosition(i, false);
+         i->setSrc(0, si->getSrc(0));
+         i->setSrc(1, bld.loadImm(NULL,
+                                  imm.reg.data.u32 + siImm->reg.data.u32));
+      }
+   }
+      break;
+
+   case OP_ABS:
+   case OP_NEG:
+   case OP_LG2:
+   case OP_RCP:
+   case OP_SQRT:
+   case OP_RSQ:
+   case OP_PRESIN:
+   case OP_SIN:
+   case OP_COS:
+   case OP_PREEX2:
+   case OP_EX2:
+      unary(i, imm);
+      break;
+   default:
+      return;
+   }
+   if (i->op != op)
+      foldCount++;
+}
+
+// =============================================================================
+
+// Merge modifier operations (ABS, NEG, NOT) into ValueRefs where allowed.
+class ModifierFolding : public Pass
+{
+private:
+   virtual bool visit(BasicBlock *);
+};
+
+bool
+ModifierFolding::visit(BasicBlock *bb)
+{
+   const Target *target = prog->getTarget();
+
+   Instruction *i, *next, *mi;
+   Modifier mod;
+
+   for (i = bb->getEntry(); i; i = next) {
+      next = i->next;
+
+      if (0 && i->op == OP_SUB) {
+         // turn "sub" into "add neg" (do we really want this ?)
+         i->op = OP_ADD;
+         i->src[0].mod = i->src[0].mod ^ Modifier(NV50_IR_MOD_NEG);
+      }
+
+      for (int s = 0; s < 3 && i->srcExists(s); ++s) {
+         mi = i->getSrc(s)->getInsn();
+         if (!mi ||
+             mi->predSrc >= 0 || mi->getDef(0)->refCount() > 8)
+            continue;
+         if (i->sType == TYPE_U32 && mi->dType == TYPE_S32) {
+            if ((i->op != OP_ADD &&
+                 i->op != OP_MUL) ||
+                (mi->op != OP_ABS &&
+                 mi->op != OP_NEG))
+               continue;
+         } else
+         if (i->sType != mi->dType) {
+            continue;
+         }
+         if ((mod = Modifier(mi->op)) == Modifier(0))
+            continue;
+         mod = mod * mi->src[0].mod;
+
+         if ((i->op == OP_ABS) || i->src[s].mod.abs()) {
+            // abs neg [abs] = abs
+            mod = mod & Modifier(~(NV50_IR_MOD_NEG | NV50_IR_MOD_ABS));
+         } else
+         if ((i->op == OP_NEG) && mod.neg()) {
+            assert(s == 0);
+            // neg as both opcode and modifier on same insn is prohibited
+            // neg neg abs = abs, neg neg = identity
+            mod = mod & Modifier(~NV50_IR_MOD_NEG);
+            i->op = mod.getOp();
+            mod = mod & Modifier(~NV50_IR_MOD_ABS);
+            if (mod == Modifier(0))
+               i->op = OP_MOV;
+         }
+
+         if (target->isModSupported(i, s, mod)) {
+            i->setSrc(s, mi->getSrc(0));
+            i->src[s].mod = i->src[s].mod * mod;
+         }
+      }
+
+      if (i->op == OP_SAT) {
+         mi = i->getSrc(0)->getInsn();
+         if (mi &&
+             mi->getDef(0)->refCount() <= 1 && target->isSatSupported(mi)) {
+            mi->saturate = 1;
+            mi->setDef(0, i->getDef(0));
+            delete_Instruction(prog, i);
+         }
+      }
+   }
+
+   return true;
+}
+
+// =============================================================================
+
+// MUL + ADD -> MAD/FMA
+// MIN/MAX(a, a) -> a, etc.
+// SLCT(a, b, const) -> cc(const) ? a : b
+// RCP(RCP(a)) -> a
+// MUL(MUL(a, b), const) -> MUL_Xconst(a, b)
+class AlgebraicOpt : public Pass
+{
+private:
+   virtual bool visit(BasicBlock *);
+
+   void handleADD(Instruction *);
+   void handleMINMAX(Instruction *);
+   void handleRCP(Instruction *);
+   void handleSLCT(Instruction *);
+   void handleLOGOP(Instruction *);
+   void handleCVT(Instruction *);
+};
+
+void
+AlgebraicOpt::handleADD(Instruction *add)
+{
+   Value *src0 = add->getSrc(0);
+   Value *src1 = add->getSrc(1);
+   Value *src;
+   int s;
+   Modifier mod[4];
+
+   if (!prog->getTarget()->isOpSupported(OP_MAD, add->dType))
+      return;
+
+   if (src0->reg.file != FILE_GPR || src1->reg.file != FILE_GPR)
+      return;
+
+   if (src0->refCount() == 1 &&
+       src0->getUniqueInsn() && src0->getUniqueInsn()->op == OP_MUL)
+      s = 0;
+   else
+   if (src1->refCount() == 1 &&
+       src1->getUniqueInsn() && src1->getUniqueInsn()->op == OP_MUL)
+      s = 1;
+   else
+      return;
+
+   if ((src0->getUniqueInsn() && src0->getUniqueInsn()->bb != add->bb) ||
+       (src1->getUniqueInsn() && src1->getUniqueInsn()->bb != add->bb))
+      return;
+
+   src = add->getSrc(s);
+
+   mod[0] = add->src[0].mod;
+   mod[1] = add->src[1].mod;
+   mod[2] = src->getUniqueInsn()->src[0].mod;
+   mod[3] = src->getUniqueInsn()->src[1].mod;
+
+   if (((mod[0] | mod[1]) | (mod[2] | mod[3])) & Modifier(~NV50_IR_MOD_NEG))
+      return;
+
+   add->op = OP_MAD;
+   add->subOp = src->getInsn()->subOp; // potentially mul-high
+
+   add->setSrc(2, add->src[s ? 0 : 1]);
+
+   add->setSrc(0, src->getInsn()->getSrc(0));
+   add->src[0].mod = mod[2] ^ mod[s];
+   add->setSrc(1, src->getInsn()->getSrc(1));
+   add->src[1].mod = mod[3];
+}
+
+void
+AlgebraicOpt::handleMINMAX(Instruction *minmax)
+{
+   Value *src0 = minmax->getSrc(0);
+   Value *src1 = minmax->getSrc(1);
+
+   if (src0 != src1 || src0->reg.file != FILE_GPR)
+      return;
+   if (minmax->src[0].mod == minmax->src[1].mod) {
+      if (minmax->src[0].mod) {
+         minmax->op = OP_CVT;
+         minmax->setSrc(1, NULL);
+      } else {
+         minmax->def[0].replace(minmax->getSrc(0), false);
+         minmax->bb->remove(minmax);
+      }
+   } else {
+      // TODO:
+      // min(x, -x) = -abs(x)
+      // min(x, -abs(x)) = -abs(x)
+      // min(x, abs(x)) = x
+      // max(x, -abs(x)) = x
+      // max(x, abs(x)) = abs(x)
+      // max(x, -x) = abs(x)
+   }
+}
+
+void
+AlgebraicOpt::handleRCP(Instruction *rcp)
+{
+   Instruction *si = rcp->getSrc(0)->getUniqueInsn();
+
+   if (si && si->op == OP_RCP) {
+      Modifier mod = rcp->src[0].mod * si->src[0].mod;
+      rcp->op = mod.getOp();
+      rcp->setSrc(0, si->getSrc(0));
+   }
+}
+
+void
+AlgebraicOpt::handleSLCT(Instruction *slct)
+{
+   if (slct->getSrc(2)->reg.file == FILE_IMMEDIATE) {
+      if (slct->getSrc(2)->asImm()->compare(slct->asCmp()->setCond, 0.0f))
+         slct->setSrc(0, slct->getSrc(1));
+   } else
+   if (slct->getSrc(0) != slct->getSrc(1)) {
+      return;
+   }
+   slct->op = OP_MOV;
+   slct->setSrc(1, NULL);
+   slct->setSrc(2, NULL);
+}
+
+void
+AlgebraicOpt::handleLOGOP(Instruction *logop)
+{
+   Value *src0 = logop->getSrc(0);
+   Value *src1 = logop->getSrc(1);
+
+   if (src0->reg.file != FILE_GPR || src1->reg.file != FILE_GPR)
+      return;
+
+   if (src0 == src1) {
+      if (logop->src[0].mod != Modifier(0) ||
+          logop->src[1].mod != Modifier(0))
+         return;
+      if (logop->op == OP_AND || logop->op == OP_OR) {
+         logop->def[0].replace(logop->getSrc(0), false);
+         delete_Instruction(prog, logop);
+      }
+   } else {
+      // try AND(SET, SET) -> SET_AND(SET)
+      Instruction *set0 = src0->getInsn();
+      Instruction *set1 = src1->getInsn();
+
+      if (!set0 || set0->fixed || !set1 || set1->fixed)
+         return;
+      if (set1->op != OP_SET) {
+         Instruction *xchg = set0;
+         set0 = set1;
+         set1 = xchg;
+         if (set1->op != OP_SET)
+            return;
+      }
+      if (set0->op != OP_SET &&
+          set0->op != OP_SET_AND &&
+          set0->op != OP_SET_OR &&
+          set0->op != OP_SET_XOR)
+         return;
+      if (set0->getDef(0)->refCount() > 1 &&
+          set1->getDef(0)->refCount() > 1)
+         return;
+      if (set0->getPredicate() || set1->getPredicate())
+         return;
+      // check that they don't source each other
+      for (int s = 0; s < 2; ++s)
+         if (set0->getSrc(s) == set1->getDef(0) ||
+             set1->getSrc(s) == set0->getDef(0))
+            return;
+
+      set0 = set0->clone(true);
+      set1 = set1->clone(false);
+      logop->bb->insertAfter(logop, set1);
+      logop->bb->insertAfter(logop, set0);
+
+      set0->dType = TYPE_U8;
+      set0->getDef(0)->reg.file = FILE_PREDICATE;
+      set0->getDef(0)->reg.size = 1;
+      set1->setSrc(2, set0->getDef(0));
+      switch (logop->op) {
+      case OP_AND: set1->op = OP_SET_AND; break;
+      case OP_OR:  set1->op = OP_SET_OR; break;
+      case OP_XOR: set1->op = OP_SET_XOR; break;
+      default:
+         assert(0);
+         break;
+      }
+      set1->setDef(0, logop->getDef(0));
+      delete_Instruction(prog, logop);
+   }
+}
+
+// F2I(NEG(SET with result 1.0f/0.0f)) -> SET with result -1/0
+void
+AlgebraicOpt::handleCVT(Instruction *cvt)
+{
+   if (cvt->sType != TYPE_F32 ||
+       cvt->dType != TYPE_S32 || cvt->src[0].mod != Modifier(0))
+      return;
+   Instruction *insn = cvt->getSrc(0)->getInsn();
+   if (!insn || insn->op != OP_NEG || insn->dType != TYPE_F32)
+      return;
+   if (insn->src[0].mod != Modifier(0))
+      return;
+   insn = insn->getSrc(0)->getInsn();
+   if (!insn || insn->op != OP_SET || insn->dType != TYPE_F32)
+      return;
+
+   Instruction *bset = insn->clone(false);
+   bset->dType = TYPE_U32;
+   bset->setDef(0, cvt->getDef(0));
+   cvt->bb->insertAfter(cvt, bset);
+   delete_Instruction(prog, cvt);
+}
+
+bool
+AlgebraicOpt::visit(BasicBlock *bb)
+{
+   Instruction *next;
+   for (Instruction *i = bb->getEntry(); i; i = next) {
+      next = i->next;
+      switch (i->op) {
+      case OP_ADD:
+         handleADD(i);
+         break;
+      case OP_RCP:
+         handleRCP(i);
+         break;
+      case OP_MIN:
+      case OP_MAX:
+         handleMINMAX(i);
+         break;
+      case OP_SLCT:
+         handleSLCT(i);
+         break;
+      case OP_AND:
+      case OP_OR:
+      case OP_XOR:
+         handleLOGOP(i);
+         break;
+      case OP_CVT:
+         handleCVT(i);
+         break;
+      default:
+         break;
+      }
+   }
+
+   return true;
+}
+
+// =============================================================================
+
+static inline void
+updateLdStOffset(Instruction *ldst, int32_t offset, Function *fn)
+{
+   if (offset != ldst->getSrc(0)->reg.data.offset) {
+      if (ldst->getSrc(0)->refCount() > 1)
+         ldst->setSrc(0, ldst->getSrc(0)->clone(fn));
+      ldst->getSrc(0)->reg.data.offset = offset;
+   }
+}
+
+// Combine loads and stores, forward stores to loads where possible.
+class MemoryOpt : public Pass
+{
+private:
+   class Record
+   {
+   public:
+      Record *next;
+      Instruction *insn;
+      const Value *rel[2];
+      const Value *base;
+      int32_t offset;
+      int8_t fileIndex;
+      uint8_t size;
+      bool locked;
+      Record *prev;
+
+      bool overlaps(const Instruction *ldst) const;
+
+      inline void link(Record **);
+      inline void unlink(Record **);
+      inline void set(const Instruction *ldst);
+   };
+
+public:
+   MemoryOpt();
+
+   Record *loads[DATA_FILE_COUNT];
+   Record *stores[DATA_FILE_COUNT];
+
+   MemoryPool recordPool;
+
+private:
+   virtual bool visit(BasicBlock *);
+   bool runOpt(BasicBlock *);
+
+   Record **getList(const Instruction *);
+
+   Record *findRecord(const Instruction *, bool load, bool& isAdjacent) const;
+
+   // merge @insn into load/store instruction from @rec
+   bool combineLd(Record *rec, Instruction *ld);
+   bool combineSt(Record *rec, Instruction *st);
+
+   bool replaceLdFromLd(Instruction *ld, Record *ldRec);
+   bool replaceLdFromSt(Instruction *ld, Record *stRec);
+   bool replaceStFromSt(Instruction *restrict st, Record *stRec);
+
+   void addRecord(Instruction *ldst);
+   void purgeRecords(Instruction *const st, DataFile);
+   void lockStores(Instruction *const ld);
+   void reset();
+
+private:
+   Record *prevRecord;
+};
+
+MemoryOpt::MemoryOpt() : recordPool(sizeof(MemoryOpt::Record), 6)
+{
+   for (int i = 0; i < DATA_FILE_COUNT; ++i) {
+      loads[i] = NULL;
+      stores[i] = NULL;
+   }
+   prevRecord = NULL;
+}
+
+void
+MemoryOpt::reset()
+{
+   for (unsigned int i = 0; i < DATA_FILE_COUNT; ++i) {
+      Record *it, *next;
+      for (it = loads[i]; it; it = next) {
+         next = it->next;
+         recordPool.release(it);
+      }
+      loads[i] = NULL;
+      for (it = stores[i]; it; it = next) {
+         next = it->next;
+         recordPool.release(it);
+      }
+      stores[i] = NULL;
+   }
+}
+
+bool
+MemoryOpt::combineLd(Record *rec, Instruction *ld)
+{
+   int32_t offRc = rec->offset;
+   int32_t offLd = ld->getSrc(0)->reg.data.offset;
+   int sizeRc = rec->size;
+   int sizeLd = typeSizeof(ld->dType);
+   int size = sizeRc + sizeLd;
+   int d, j;
+
+   // only VFETCH can do a 96 byte load
+   if (ld->op != OP_VFETCH && size == 12)
+      return false;
+   // no unaligned loads
+   if (((size == 0x8) && (MIN2(offLd, offRc) & 0x7)) ||
+       ((size == 0xc) && (MIN2(offLd, offRc) & 0xf)))
+      return false;
+
+   assert(sizeRc + sizeLd <= 16 && offRc != offLd);
+
+   for (j = 0; sizeRc; sizeRc -= rec->insn->getDef(j)->reg.size, ++j);
+
+   if (offLd < offRc) {
+      int sz;
+      for (sz = 0, d = 0; sz < sizeLd; sz += ld->getDef(d)->reg.size, ++d);
+      // d: nr of definitions in ld
+      // j: nr of definitions in rec->insn, move:
+      for (d = d + j - 1; j > 0; --j, --d)
+         rec->insn->setDef(d, rec->insn->getDef(j - 1));
+
+      if (rec->insn->getSrc(0)->refCount() > 1)
+         rec->insn->setSrc(0, rec->insn->getSrc(0)->clone(func));
+      rec->offset = rec->insn->getSrc(0)->reg.data.offset = offLd;
+
+      d = 0;
+   } else {
+      d = j;
+   }
+   // move definitions of @ld to @rec->insn
+   for (j = 0; sizeLd; ++j, ++d) {
+      sizeLd -= ld->getDef(j)->reg.size;
+      rec->insn->setDef(d, ld->getDef(j));
+   }
+
+   rec->size = size;
+   rec->insn->setType(typeOfSize(size));
+
+   delete_Instruction(prog, ld);
+
+   return true;
+}
+
+bool
+MemoryOpt::combineSt(Record *rec, Instruction *st)
+{
+   int32_t offRc = rec->offset;
+   int32_t offSt = st->getSrc(0)->reg.data.offset;
+   int sizeRc = rec->size;
+   int sizeSt = typeSizeof(st->dType);
+   int s = sizeSt / 4;
+   int size = sizeRc + sizeSt;
+   int j, k;
+   Value *src[4]; // no modifiers in ValueRef allowed for st
+   Value *extra[3];
+
+   if (size == 12) // XXX: check if EXPORT a[] can do this after all
+      return false;
+   if (size == 8 && MIN2(offRc, offSt) & 0x7)
+      return false;
+
+   st->takeExtraSources(0, extra); // save predicate and indirect address
+
+   if (offRc < offSt) {
+      // save values from @st
+      for (s = 0; sizeSt; ++s) {
+         sizeSt -= st->getSrc(s + 1)->reg.size;
+         src[s] = st->getSrc(s + 1);
+      }
+      // set record's values as low sources of @st
+      for (j = 1; sizeRc; ++j) {
+         sizeRc -= st->getSrc(j)->reg.size;
+         st->setSrc(j, rec->insn->getSrc(j));
+      }
+      // set saved values as high sources of @st
+      for (k = j, j = 0; j < s; ++j)
+         st->setSrc(k++, src[j]);
+
+      updateLdStOffset(st, offRc, func);
+   } else {
+      for (j = 1; sizeSt; ++j)
+         sizeSt -= st->getSrc(j)->reg.size;
+      for (s = 1; sizeRc; ++j, ++s) {
+         sizeRc -= rec->insn->getSrc(s)->reg.size;
+         st->setSrc(j, rec->insn->getSrc(s));
+      }
+      rec->offset = offSt;
+   }
+   st->putExtraSources(0, extra); // restore pointer and predicate
+
+   delete_Instruction(prog, rec->insn);
+   rec->insn = st;
+   rec->size = size;
+   rec->insn->setType(typeOfSize(size));
+   return true;
+}
+
+void
+MemoryOpt::Record::set(const Instruction *ldst)
+{
+   const Symbol *mem = ldst->getSrc(0)->asSym();
+   fileIndex = mem->reg.fileIndex;
+   rel[0] = ldst->getIndirect(0, 0);
+   rel[1] = ldst->getIndirect(0, 1);
+   offset = mem->reg.data.offset;
+   base = mem->getBase();
+   size = typeSizeof(ldst->sType);
+}
+
+void
+MemoryOpt::Record::link(Record **list)
+{
+   next = *list;
+   if (next)
+      next->prev = this;
+   prev = NULL;
+   *list = this;
+}
+
+void
+MemoryOpt::Record::unlink(Record **list)
+{
+   if (next)
+      next->prev = prev;
+   if (prev)
+      prev->next = next;
+   else
+      *list = next;
+}
+
+MemoryOpt::Record **
+MemoryOpt::getList(const Instruction *insn)
+{
+   if (insn->op == OP_LOAD || insn->op == OP_VFETCH)
+      return &loads[insn->src[0].getFile()];
+   return &stores[insn->src[0].getFile()];
+}
+
+void
+MemoryOpt::addRecord(Instruction *i)
+{
+   Record **list = getList(i);
+   Record *it = reinterpret_cast<Record *>(recordPool.allocate());
+
+   it->link(list);
+   it->set(i);
+   it->insn = i;
+   it->locked = false;
+}
+
+MemoryOpt::Record *
+MemoryOpt::findRecord(const Instruction *insn, bool load, bool& isAdj) const
+{
+   const Symbol *sym = insn->getSrc(0)->asSym();
+   const int size = typeSizeof(insn->sType);
+   Record *rec = NULL;
+   Record *it = load ? loads[sym->reg.file] : stores[sym->reg.file];
+
+   for (; it; it = it->next) {
+      if (it->locked && insn->op != OP_LOAD)
+         continue;
+      if ((it->offset >> 4) != (sym->reg.data.offset >> 4) ||
+          it->rel[0] != insn->getIndirect(0, 0) ||
+          it->fileIndex != sym->reg.fileIndex ||
+          it->rel[1] != insn->getIndirect(0, 1))
+         continue;
+
+      if (it->offset < sym->reg.data.offset) {
+         if (it->offset + it->size >= sym->reg.data.offset) {
+            isAdj = (it->offset + it->size == sym->reg.data.offset);
+            if (!isAdj)
+               return it;
+            if (!(it->offset & 0x7))
+               rec = it;
+         }
+      } else {
+         isAdj = it->offset != sym->reg.data.offset;
+         if (size <= it->size && !isAdj)
+            return it;
+         else
+         if (!(sym->reg.data.offset & 0x7))
+            if (it->offset - size <= sym->reg.data.offset)
+               rec = it;
+      }
+   }
+   return rec;
+}
+
+bool
+MemoryOpt::replaceLdFromSt(Instruction *ld, Record *rec)
+{
+   Instruction *st = rec->insn;
+   int32_t offSt = rec->offset;
+   int32_t offLd = ld->getSrc(0)->reg.data.offset;
+   int d, s;
+
+   for (s = 1; offSt != offLd && st->srcExists(s); ++s)
+      offSt += st->getSrc(s)->reg.size;
+   if (offSt != offLd)
+      return false;
+
+   for (d = 0; ld->defExists(d) && st->srcExists(s); ++d, ++s) {
+      if (ld->getDef(d)->reg.size != st->getSrc(s)->reg.size)
+         return false;
+      if (st->getSrc(s)->reg.file != FILE_GPR)
+         return false;
+      ld->def[d].replace(st->getSrc(s), false);
+   }
+   ld->bb->remove(ld);
+   return true;
+}
+
+bool
+MemoryOpt::replaceLdFromLd(Instruction *ldE, Record *rec)
+{
+   Instruction *ldR = rec->insn;
+   int32_t offR = rec->offset;
+   int32_t offE = ldE->getSrc(0)->reg.data.offset;
+   int dR, dE;
+
+   assert(offR <= offE);
+   for (dR = 0; offR < offE && ldR->defExists(dR); ++dR)
+      offR += ldR->getDef(dR)->reg.size;
+   if (offR != offE)
+      return false;
+
+   for (dE = 0; ldE->defExists(dE) && ldR->defExists(dR); ++dE, ++dR) {
+      if (ldE->getDef(dE)->reg.size != ldR->getDef(dR)->reg.size)
+         return false;
+      ldE->def[dE].replace(ldR->getDef(dR), false);
+   }
+
+   delete_Instruction(prog, ldE);
+   return true;
+}
+
+bool
+MemoryOpt::replaceStFromSt(Instruction *restrict st, Record *rec)
+{
+   const Instruction *const ri = rec->insn;
+   Value *extra[3];
+
+   int32_t offS = st->getSrc(0)->reg.data.offset;
+   int32_t offR = rec->offset;
+   int32_t endS = offS + typeSizeof(st->dType);
+   int32_t endR = offR + typeSizeof(ri->dType);
+
+   rec->size = MAX2(endS, endR) - MIN2(offS, offR);
+
+   st->takeExtraSources(0, extra);
+
+   if (offR < offS) {
+      Value *vals[4];
+      int s, n;
+      int k = 0;
+      // get non-replaced sources of ri
+      for (s = 1; offR < offS; offR += ri->getSrc(s)->reg.size, ++s)
+         vals[k++] = ri->getSrc(s);
+      n = s;
+      // get replaced sources of st
+      for (s = 1; st->srcExists(s); offS += st->getSrc(s)->reg.size, ++s)
+         vals[k++] = st->getSrc(s);
+      // skip replaced sources of ri
+      for (s = n; offR < endS; offR += ri->getSrc(s)->reg.size, ++s);
+      // get non-replaced sources after values covered by st
+      for (; offR < endR; offR += ri->getSrc(s)->reg.size, ++s)
+         vals[k++] = ri->getSrc(s);
+      for (s = 0; s < k; ++s)
+         st->setSrc(s + 1, vals[s]);
+      st->setSrc(0, ri->getSrc(0));
+   } else
+   if (endR > endS) {
+      int j, s;
+      for (j = 1; offR < endS; offR += ri->getSrc(j++)->reg.size);
+      for (s = 1; offS < endS; offS += st->getSrc(s++)->reg.size);
+      for (; offR < endR; offR += ri->getSrc(j++)->reg.size)
+         st->setSrc(s++, ri->getSrc(j));
+   }
+   st->putExtraSources(0, extra);
+
+   delete_Instruction(prog, rec->insn);
+
+   rec->insn = st;
+   rec->offset = st->getSrc(0)->reg.data.offset;
+
+   st->setType(typeOfSize(rec->size));
+
+   return true;
+}
+
+bool
+MemoryOpt::Record::overlaps(const Instruction *ldst) const
+{
+   Record that;
+   that.set(ldst);
+
+   if (this->fileIndex != that.fileIndex)
+      return false;
+
+   if (this->rel[0] || that.rel[0])
+      return this->base == that.base;
+   return
+      (this->offset < that.offset + that.size) &&
+      (this->offset + this->size > that.offset);
+}
+
+// We must not eliminate stores that affect the result of @ld if
+// we find later stores to the same location, and we may no longer
+// merge them with later stores.
+// The stored value can, however, still be used to determine the value
+// returned by future loads.
+void
+MemoryOpt::lockStores(Instruction *const ld)
+{
+   for (Record *r = stores[ld->src[0].getFile()]; r; r = r->next)
+      if (!r->locked && r->overlaps(ld))
+         r->locked = true;
+}
+
+// Prior loads from the location of @st are no longer valid.
+// Stores to the location of @st may no longer be used to derive
+// the value at it nor be coalesced into later stores.
+void
+MemoryOpt::purgeRecords(Instruction *const st, DataFile f)
+{
+   if (st)
+      f = st->src[0].getFile();
+
+   for (Record *r = loads[f]; r; r = r->next)
+      if (!st || r->overlaps(st))
+         r->unlink(&loads[f]);
+
+   for (Record *r = stores[f]; r; r = r->next)
+      if (!st || r->overlaps(st))
+         r->unlink(&stores[f]);
+}
+
+bool
+MemoryOpt::visit(BasicBlock *bb)
+{
+   bool ret = runOpt(bb);
+   // Run again, one pass won't combine 4 32 bit ld/st to a single 128 bit ld/st
+   // where 96 bit memory operations are forbidden.
+   if (ret)
+      ret = runOpt(bb);
+   return ret;
+}
+
+bool
+MemoryOpt::runOpt(BasicBlock *bb)
+{
+   Instruction *ldst, *next;
+   Record *rec;
+   bool isAdjacent = true;
+
+   for (ldst = bb->getEntry(); ldst; ldst = next) {
+      bool keep = true;
+      bool isLoad = true;
+      next = ldst->next;
+
+      if (ldst->op == OP_LOAD || ldst->op == OP_VFETCH) {
+         if (ldst->isDead()) {
+            // might have been produced by earlier optimization
+            delete_Instruction(prog, ldst);
+            continue;
+         }
+      } else
+      if (ldst->op == OP_STORE || ldst->op == OP_EXPORT) {
+         isLoad = false;
+      } else {
+         // TODO: maybe have all fixed ops act as barrier ?
+         if (ldst->op == OP_CALL) {
+            purgeRecords(NULL, FILE_MEMORY_LOCAL);
+            purgeRecords(NULL, FILE_MEMORY_GLOBAL);
+            purgeRecords(NULL, FILE_MEMORY_SHARED);
+            purgeRecords(NULL, FILE_SHADER_OUTPUT);
+         } else
+         if (ldst->op == OP_EMIT || ldst->op == OP_RESTART) {
+            purgeRecords(NULL, FILE_SHADER_OUTPUT);
+         }
+         continue;
+      }
+      if (ldst->getPredicate()) // TODO: handle predicated ld/st
+         continue;
+
+      if (isLoad) {
+         DataFile file = ldst->src[0].getFile();
+
+         // if ld l[]/g[] look for previous store to eliminate the reload
+         if (file == FILE_MEMORY_GLOBAL || file == FILE_MEMORY_LOCAL) {
+            // TODO: shared memory ?
+            rec = findRecord(ldst, false, isAdjacent);
+            if (rec && !isAdjacent)
+               keep = !replaceLdFromSt(ldst, rec);
+         }
+
+         // or look for ld from the same location and replace this one
+         rec = keep ? findRecord(ldst, true, isAdjacent) : NULL;
+         if (rec) {
+            if (!isAdjacent)
+               keep = !replaceLdFromLd(ldst, rec);
+            else
+               // or combine a previous load with this one
+               keep = !combineLd(rec, ldst);
+         }
+         if (keep)
+            lockStores(ldst);
+      } else {
+         rec = findRecord(ldst, false, isAdjacent);
+         if (rec) {
+            if (!isAdjacent)
+               keep = !replaceStFromSt(ldst, rec);
+            else
+               keep = !combineSt(rec, ldst);
+         }
+         if (keep)
+            purgeRecords(ldst, DATA_FILE_COUNT);
+      }
+      if (keep)
+         addRecord(ldst);
+   }
+   reset();
+
+   return true;
+}
+
+// =============================================================================
+
+// Turn control flow into predicated instructions (after register allocation !).
+// TODO:
+// Could move this to before register allocation on NVC0 and also handle nested
+// constructs.
+class FlatteningPass : public Pass
+{
+private:
+   virtual bool visit(BasicBlock *);
+
+   bool tryPredicateConditional(BasicBlock *);
+   void predicateInstructions(BasicBlock *, Value *pred, CondCode cc);
+   void tryPropagateBranch(BasicBlock *);
+   inline bool isConstantCondition(Value *pred);
+   inline bool mayPredicate(const Instruction *, const Value *pred) const;
+   inline void removeFlow(Instruction *);
+};
+
+bool
+FlatteningPass::isConstantCondition(Value *pred)
+{
+   Instruction *insn = pred->getUniqueInsn();
+   assert(insn);
+   if (insn->op != OP_SET || insn->srcExists(2))
+      return false;
+
+   for (int s = 0; s < 2 && insn->srcExists(s); ++s) {
+      Instruction *ld = insn->getSrc(s)->getUniqueInsn();
+      DataFile file;
+      if (ld) {
+         if (ld->op != OP_MOV && ld->op != OP_LOAD)
+            return false;
+         if (ld->src[0].isIndirect(0))
+            return false;
+         file = ld->src[0].getFile();
+      } else {
+         file = insn->src[s].getFile();
+         // catch $r63 on NVC0
+         if (file == FILE_GPR && insn->getSrc(s)->reg.data.id > prog->maxGPR)
+            file = FILE_IMMEDIATE;
+      }
+      if (file != FILE_IMMEDIATE && file != FILE_MEMORY_CONST)
+         return false;
+   }
+   return true;
+}
+
+void
+FlatteningPass::removeFlow(Instruction *insn)
+{
+   FlowInstruction *term = insn ? insn->asFlow() : NULL;
+   if (!term)
+      return;
+   Graph::Edge::Type ty = term->bb->cfg.outgoing().getType();
+
+   if (term->op == OP_BRA) {
+      // TODO: this might get more difficult when we get arbitrary BRAs
+      if (ty == Graph::Edge::CROSS || ty == Graph::Edge::BACK)
+         return;
+   } else
+   if (term->op != OP_JOIN)
+      return;
+
+   delete_Instruction(prog, term);
+
+   Value *pred = term->getPredicate();
+
+   if (pred && pred->refCount() == 0) {
+      Instruction *pSet = pred->getUniqueInsn();
+      pred->join->reg.data.id = -1; // deallocate
+      if (pSet->isDead())
+         delete_Instruction(prog, pSet);
+   }
+}
+
+void
+FlatteningPass::predicateInstructions(BasicBlock *bb, Value *pred, CondCode cc)
+{
+   for (Instruction *i = bb->getEntry(); i; i = i->next) {
+      if (i->isNop())
+         continue;
+      assert(!i->getPredicate());
+      i->setPredicate(cc, pred);
+   }
+   removeFlow(bb->getExit());
+}
+
+bool
+FlatteningPass::mayPredicate(const Instruction *insn, const Value *pred) const
+{
+   if (insn->isPseudo())
+      return true;
+   // TODO: calls where we don't know which registers are modified
+
+   if (!prog->getTarget()->mayPredicate(insn, pred))
+      return false;
+   for (int d = 0; insn->defExists(d); ++d)
+      if (insn->getDef(d)->equals(pred))
+         return false;
+   return true;
+}
+
+// If we conditionally skip over or to a branch instruction, replace it.
+// NOTE: We do not update the CFG anymore here !
+void
+FlatteningPass::tryPropagateBranch(BasicBlock *bb)
+{
+   BasicBlock *bf = NULL;
+   unsigned int i;
+
+   if (bb->cfg.outgoingCount() != 2)
+      return;
+   if (!bb->getExit() || bb->getExit()->op != OP_BRA)
+      return;
+   Graph::EdgeIterator ei = bb->cfg.outgoing();
+
+   for (i = 0; !ei.end(); ++i, ei.next()) {
+      bf = BasicBlock::get(ei.getNode());
+      if (bf->getInsnCount() == 1)
+         break;
+   }
+   if (ei.end() || !bf->getExit())
+      return;
+   FlowInstruction *bra = bb->getExit()->asFlow();
+   FlowInstruction *rep = bf->getExit()->asFlow();
+
+   if (rep->getPredicate())
+      return;
+   if (rep->op != OP_BRA &&
+       rep->op != OP_JOIN &&
+       rep->op != OP_EXIT)
+      return;
+
+   bra->op = rep->op;
+   bra->target.bb = rep->target.bb;
+   if (i) // 2nd out block means branch not taken
+      bra->cc = inverseCondCode(bra->cc);
+   bf->remove(rep);
+}
+
+bool
+FlatteningPass::visit(BasicBlock *bb)
+{
+   if (tryPredicateConditional(bb))
+      return true;
+
+   // try to attach join to previous instruction
+   Instruction *insn = bb->getExit();
+   if (insn && insn->op == OP_JOIN && !insn->getPredicate()) {
+      insn = insn->prev;
+      if (insn && !insn->getPredicate() && !insn->asFlow() && !insn->isNop()) {
+         insn->join = 1;
+         bb->remove(bb->getExit());
+         return true;
+      }
+   }
+
+   tryPropagateBranch(bb);
+
+   return true;
+}
+
+bool
+FlatteningPass::tryPredicateConditional(BasicBlock *bb)
+{
+   BasicBlock *bL = NULL, *bR = NULL;
+   unsigned int nL = 0, nR = 0, limit = 12;
+   Instruction *insn;
+   unsigned int mask;
+
+   mask = bb->initiatesSimpleConditional();
+   if (!mask)
+      return false;
+
+   assert(bb->getExit());
+   Value *pred = bb->getExit()->getPredicate();
+   assert(pred);
+
+   if (isConstantCondition(pred))
+      limit = 4;
+
+   Graph::EdgeIterator ei = bb->cfg.outgoing();
+
+   if (mask & 1) {
+      bL = BasicBlock::get(ei.getNode());
+      for (insn = bL->getEntry(); insn; insn = insn->next, ++nL)
+         if (!mayPredicate(insn, pred))
+            return false;
+      if (nL > limit)
+         return false; // too long, do a real branch
+   }
+   ei.next();
+
+   if (mask & 2) {
+      bR = BasicBlock::get(ei.getNode());
+      for (insn = bR->getEntry(); insn; insn = insn->next, ++nR)
+         if (!mayPredicate(insn, pred))
+            return false;
+      if (nR > limit)
+         return false; // too long, do a real branch
+   }
+
+   if (bL)
+      predicateInstructions(bL, pred, bb->getExit()->cc);
+   if (bR)
+      predicateInstructions(bR, pred, inverseCondCode(bb->getExit()->cc));
+
+   if (bb->joinAt) {
+      bb->remove(bb->joinAt);
+      bb->joinAt = NULL;
+   }
+   removeFlow(bb->getExit()); // delete the branch/join at the fork point
+
+   // remove potential join operations at the end of the conditional
+   if (prog->getTarget()->joinAnterior) {
+      bb = BasicBlock::get((bL ? bL : bR)->cfg.outgoing().getNode());
+      if (bb->getEntry() && bb->getEntry()->op == OP_JOIN)
+         removeFlow(bb->getEntry());
+   }
+
+   return true;
+}
+
+// =============================================================================
+
+// Common subexpression elimination. Stupid O^2 implementation.
+class LocalCSE : public Pass
+{
+private:
+   virtual bool visit(BasicBlock *);
+
+   inline bool tryReplace(Instruction **, Instruction *);
+
+   DLList ops[OP_LAST + 1];
+};
+
+class GlobalCSE : public Pass
+{
+private:
+   virtual bool visit(BasicBlock *);
+};
+
+bool
+Instruction::isActionEqual(const Instruction *that) const
+{
+   if (this->op != that->op ||
+       this->dType != that->dType ||
+       this->sType != that->sType)
+      return false;
+   if (this->cc != that->cc)
+      return false;
+
+   if (this->asTex()) {
+      if (memcmp(&this->asTex()->tex,
+                 &that->asTex()->tex,
+                 sizeof(this->asTex()->tex)))
+         return false;
+   } else
+   if (this->asCmp()) {
+      if (this->asCmp()->setCond != that->asCmp()->setCond)
+         return false;
+   } else
+   if (this->asFlow()) {
+      return false;
+   } else {
+      if (this->atomic != that->atomic ||
+          this->ipa != that->ipa ||
+          this->lanes != that->lanes ||
+          this->perPatch != that->perPatch)
+         return false;
+      if (this->postFactor != that->postFactor)
+         return false;
+   }
+
+   if (this->subOp != that->subOp ||
+       this->saturate != that->saturate ||
+       this->rnd != that->rnd ||
+       this->ftz != that->ftz ||
+       this->dnz != that->dnz ||
+       this->cache != that->cache)
+      return false;
+
+   return true;
+}
+
+bool
+Instruction::isResultEqual(const Instruction *that) const
+{
+   unsigned int d, s;
+
+   // NOTE: location of discard only affects tex with liveOnly and quadops
+   if (!this->defExists(0) && this->op != OP_DISCARD)
+      return false;
+
+   if (!isActionEqual(that))
+      return false;
+
+   if (this->predSrc != that->predSrc)
+      return false;
+
+   for (d = 0; this->defExists(d); ++d) {
+      if (!that->defExists(d) ||
+          !this->getDef(d)->equals(that->getDef(d), false))
+         return false;
+   }
+   if (that->defExists(d))
+      return false;
+
+   for (s = 0; this->srcExists(s); ++s) {
+      if (!that->srcExists(s))
+         return false;
+      if (this->src[s].mod != that->src[s].mod)
+         return false;
+      if (!this->getSrc(s)->equals(that->getSrc(s), true))
+         return false;
+   }
+   if (that->srcExists(s))
+      return false;
+
+   if (op == OP_LOAD || op == OP_VFETCH) {
+      switch (src[0].getFile()) {
+      case FILE_MEMORY_CONST:
+      case FILE_SHADER_INPUT:
+         return true;
+      default:
+         return false;
+      }
+   }
+
+   return true;
+}
+
+// pull through common expressions from different in-blocks
+bool
+GlobalCSE::visit(BasicBlock *bb)
+{
+   Instruction *phi, *next, *ik;
+   int s;
+
+   for (phi = bb->getPhi(); phi && phi->op == OP_PHI; phi = next) {
+      next = phi->next;
+      if (phi->getSrc(0)->refCount() > 1)
+         continue;
+      ik = phi->getSrc(0)->getInsn();
+      for (s = 1; phi->srcExists(s); ++s) {
+         if (phi->getSrc(s)->refCount() > 1)
+            break;
+         if (!phi->getSrc(s)->getInsn()->isResultEqual(ik))
+            break;
+      }
+      if (!phi->srcExists(s)) {
+         Instruction *entry = bb->getEntry();
+         ik->bb->remove(ik);
+         if (!entry || entry->op != OP_JOIN)
+            bb->insertHead(ik);
+         else
+            bb->insertAfter(entry, ik);
+         ik->setDef(0, phi->getDef(0));
+         delete_Instruction(prog, phi);
+      }
+   }
+
+   return true;
+}
+
+bool
+LocalCSE::tryReplace(Instruction **ptr, Instruction *i)
+{
+   Instruction *old = *ptr;
+   if (!old->isResultEqual(i))
+      return false;
+   for (int d = 0; old->defExists(d); ++d)
+      old->def[d].replace(i->getDef(d), false);
+   delete_Instruction(prog, old);
+   *ptr = NULL;
+   return true;
+}
+
+bool
+LocalCSE::visit(BasicBlock *bb)
+{
+   unsigned int replaced;
+
+   do {
+      Instruction *ir, *next;
+
+      replaced = 0;
+
+      // will need to know the order of instructions
+      int serial = 0;
+      for (ir = bb->getEntry(); ir; ir = ir->next)
+         ir->serial = serial++;
+
+      for (ir = bb->getEntry(); ir; ir = next) {
+         int s;
+         Value *src = NULL;
+
+         next = ir->next;
+
+         if (ir->fixed) {
+            ops[ir->op].insert(ir);
+            continue;
+         }
+
+         for (s = 0; ir->srcExists(s); ++s)
+            if (ir->getSrc(s)->asLValue())
+               if (!src || ir->getSrc(s)->refCount() < src->refCount())
+                  src = ir->getSrc(s);
+
+         if (src) {
+            for (ValueRef::Iterator refs = src->uses->iterator(); !refs.end();
+                 refs.next()) {
+               Instruction *ik = refs.get()->getInsn();
+               if (ik->serial < ir->serial && ik->bb == ir->bb)
+                  if (tryReplace(&ir, ik))
+                     break;
+            }
+         } else {
+            DLLIST_FOR_EACH(&ops[ir->op], iter)
+            {
+               Instruction *ik = reinterpret_cast<Instruction *>(iter.get());
+               if (tryReplace(&ir, ik))
+                  break;
+            }
+         }
+
+         if (ir)
+            ops[ir->op].insert(ir);
+         else
+            ++replaced;
+      }
+      for (unsigned int i = 0; i <= OP_LAST; ++i)
+         ops[i].clear();
+
+   } while (replaced);
+
+   return true;
+}
+
+// =============================================================================
+
+// Remove computations of unused values.
+class DeadCodeElim : public Pass
+{
+public:
+   bool buryAll(Program *);
+
+private:
+   virtual bool visit(BasicBlock *);
+
+   void checkSplitLoad(Instruction *ld); // for partially dead loads
+
+   unsigned int deadCount;
+};
+
+bool
+DeadCodeElim::buryAll(Program *prog)
+{
+   do {
+      deadCount = 0;
+      if (!this->run(prog, false, false))
+         return false;
+   } while (deadCount);
+
+   return true;
+}
+
+bool
+DeadCodeElim::visit(BasicBlock *bb)
+{
+   Instruction *next;
+
+   for (Instruction *i = bb->getFirst(); i; i = next) {
+      next = i->next;
+      if (i->isDead()) {
+         ++deadCount;
+         delete_Instruction(prog, i);
+      } else
+      if (i->defExists(1) && (i->op == OP_VFETCH || i->op == OP_LOAD)) {
+         checkSplitLoad(i);
+      }
+   }
+   return true;
+}
+
+void
+DeadCodeElim::checkSplitLoad(Instruction *ld1)
+{
+   Instruction *ld2 = NULL; // can get at most 2 loads
+   Value *def1[4];
+   Value *def2[4];
+   int32_t addr1, addr2;
+   int32_t size1, size2;
+   int d, n1, n2;
+   uint32_t mask = 0xffffffff;
+
+   for (d = 0; ld1->defExists(d); ++d)
+      if (!ld1->getDef(d)->refCount() && ld1->getDef(d)->reg.data.id < 0)
+         mask &= ~(1 << d);
+   if (mask == 0xffffffff)
+      return;
+
+   addr1 = ld1->getSrc(0)->reg.data.offset;
+   n1 = n2 = 0;
+   size1 = size2 = 0;
+   for (d = 0; ld1->defExists(d); ++d) {
+      if (mask & (1 << d)) {
+         if (size1 && (addr1 & 0x7))
+            break;
+         def1[n1] = ld1->getDef(d);
+         size1 += def1[n1++]->reg.size;
+      } else
+      if (!n1) {
+         addr1 += ld1->getDef(d)->reg.size;
+      } else {
+         break;
+      }
+   }
+   for (addr2 = addr1 + size1; ld1->defExists(d); ++d) {
+      if (mask & (1 << d)) {
+         def2[n2] = ld1->getDef(d);
+         size2 += def2[n2++]->reg.size;
+      } else {
+         assert(!n2);
+         addr2 += ld1->getDef(d)->reg.size;
+      }
+   }
+
+   updateLdStOffset(ld1, addr1, func);
+   ld1->setType(typeOfSize(size1));
+   for (d = 0; d < 4; ++d)
+      ld1->setDef(d, (d < n1) ? def1[d] : NULL);
+
+   if (!n2)
+      return;
+
+   ld2 = ld1->clone(false);
+   updateLdStOffset(ld2, addr2, func);
+   ld2->setType(typeOfSize(size2));
+   for (d = 0; d < 4; ++d)
+      ld2->setDef(d, (d < n2) ? def2[d] : NULL);
+
+   ld1->bb->insertAfter(ld1, ld2);
+}
+
+// =============================================================================
+
+#define RUN_PASS(l, n, f)                       \
+   if (level >= (l)) {                          \
+      if (dbgFlags & NV50_IR_DEBUG_VERBOSE)     \
+         INFO("PEEPHOLE: %s\n", #n);            \
+      n pass;                                   \
+      if (!pass.f(this))                        \
+         return false;                          \
+   }
+
+bool
+Program::optimizeSSA(int level)
+{
+   RUN_PASS(1, DeadCodeElim, buryAll);
+   RUN_PASS(1, CopyPropagation, run);
+   RUN_PASS(2, GlobalCSE, run);
+   RUN_PASS(1, LocalCSE, run);
+   RUN_PASS(2, AlgebraicOpt, run);
+   RUN_PASS(2, ModifierFolding, run); // before load propagation -> less checks
+   RUN_PASS(1, ConstantFolding, foldAll);
+   RUN_PASS(1, LoadPropagation, run);
+   RUN_PASS(2, MemoryOpt, run);
+   RUN_PASS(2, LocalCSE, run);
+   RUN_PASS(0, DeadCodeElim, buryAll);
+   return true;
+}
+
+bool
+Program::optimizePostRA(int level)
+{
+   RUN_PASS(2, FlatteningPass, run);
+   return true;
+}
+
+}
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_print.cpp b/src/gallium/drivers/nv50/codegen/nv50_ir_print.cpp
new file mode 100644
index 00000000000..b5ca3814098
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_print.cpp
@@ -0,0 +1,558 @@
+
+#include "nv50_ir.h"
+#include "nv50_ir_target.h"
+
+namespace nv50_ir {
+
+enum TextStyle
+{
+   TXT_DEFAULT,
+   TXT_GPR,
+   TXT_REGISTER,
+   TXT_FLAGS,
+   TXT_MEM,
+   TXT_IMMD,
+   TXT_BRA,
+   TXT_INSN
+};
+
+static const char *colour[8] =
+{
+#if 1
+   "\x1b[00m",
+   "\x1b[34m",
+   "\x1b[35m",
+   "\x1b[35m",
+   "\x1b[36m",
+   "\x1b[33m",
+   "\x1b[37m",
+   "\x1b[32m"
+#else
+   "", "", "", "", "", "", "", ""
+#endif
+};
+
+const char *operationStr[OP_LAST + 1] =
+{
+   "nop",
+   "phi",
+   "union",
+   "split",
+   "merge",
+   "consec",
+   "mov",
+   "ld",
+   "st",
+   "add",
+   "sub",
+   "mul",
+   "div",
+   "mod",
+   "mad",
+   "fma",
+   "sad",
+   "abs",
+   "neg",
+   "not",
+   "and",
+   "or",
+   "xor",
+   "shl",
+   "shr",
+   "max",
+   "min",
+   "sat",
+   "ceil",
+   "floor",
+   "trunc",
+   "cvt",
+   "set and",
+   "set or",
+   "set xor",
+   "set",
+   "selp",
+   "slct",
+   "rcp",
+   "rsq",
+   "lg2",
+   "sin",
+   "cos",
+   "ex2",
+   "exp",
+   "log",
+   "presin",
+   "preex2",
+   "sqrt",
+   "pow",
+   "bra",
+   "call",
+   "ret",
+   "cont",
+   "break",
+   "preret",
+   "precont",
+   "prebreak",
+   "brkpt",
+   "joinat",
+   "join",
+   "discard",
+   "exit",
+   "barrier",
+   "vfetch",
+   "pfetch",
+   "export",
+   "linterp",
+   "pinterp",
+   "emit",
+   "restart",
+   "tex",
+   "texbias",
+   "texlod",
+   "texfetch",
+   "texquery",
+   "texgrad",
+   "texgather",
+   "texcsaa",
+   "suld",
+   "sust",
+   "dfdx",
+   "dfdy",
+   "rdsv",
+   "wrsv",
+   "pixld",
+   "quadop",
+   "quadon",
+   "quadpop",
+   "popcnt",
+   "insbf",
+   "extbf",
+   "(invalid)"
+};
+
+static const char *DataTypeStr[] =
+{
+   "-",
+   "u8", "s8",
+   "u16", "s16",
+   "u32", "s32",
+   "u64", "s64",
+   "f16", "f32", "f64",
+   "b96", "b128"
+};
+
+static const char *RoundModeStr[] =
+{
+   "", "rm", "rz", "rp", "rni", "rmi", "rzi", "rpi"
+};
+
+static const char *CondCodeStr[] =
+{
+   "never",
+   "lt",
+   "eq",
+   "le",
+   "gt",
+   "ne",
+   "ge",
+   "",
+   "(invalid)",
+   "ltu",
+   "equ",
+   "leu",
+   "gtu",
+   "neu",
+   "geu",
+   "",
+   "no",
+   "nc",
+   "ns",
+   "na",
+   "a",
+   "s",
+   "c",
+   "o"
+};
+
+static const char *SemanticStr[SV_LAST + 1] =
+{
+   "POSITION",
+   "VERTEX_ID",
+   "INSTANCE_ID",
+   "INVOCATION_ID",
+   "PRIMITIVE_ID",
+   "VERTEX_COUNT",
+   "LAYER",
+   "VIEWPORT_INDEX",
+   "Y_DIR",
+   "FACE",
+   "POINT_SIZE",
+   "POINT_COORD",
+   "CLIP_DISTANCE",
+   "SAMPLE_INDEX",
+   "TESS_FACTOR",
+   "TESS_COORD",
+   "TID",
+   "CTAID",
+   "NTID",
+   "GRIDID",
+   "NCTAID",
+   "LANEID",
+   "PHYSID",
+   "NPHYSID",
+   "CLOCK",
+   "LBASE",
+   "SBASE",
+   "?",
+   "(INVALID)"
+};
+
+#define PRINT(args...)                                \
+   do {                                               \
+      pos += snprintf(&buf[pos], size - pos, args);   \
+   } while(0)
+
+#define SPACE_PRINT(cond, args...)                      \
+   do {                                                 \
+      if (cond)                                         \
+         buf[pos++] = ' ';                              \
+      pos += snprintf(&buf[pos], size - pos, args);     \
+   } while(0)
+
+#define SPACE()                                    \
+   do {                                            \
+      if (pos < size)                              \
+         buf[pos++] = ' ';                         \
+   } while(0)
+
+int Modifier::print(char *buf, size_t size) const
+{
+   size_t pos = 0;
+
+   if (bits)
+      PRINT("%s", colour[TXT_INSN]);
+
+   size_t base = pos;
+
+   if (bits & NV50_IR_MOD_NOT)
+      PRINT("not");
+   if (bits & NV50_IR_MOD_SAT)
+      SPACE_PRINT(pos > base && pos < size, "sat");
+   if (bits & NV50_IR_MOD_NEG)
+      SPACE_PRINT(pos > base && pos < size, "neg");
+   if (bits & NV50_IR_MOD_ABS)
+      SPACE_PRINT(pos > base && pos < size, "abs");
+
+   return pos;
+}
+   
+int LValue::print(char *buf, size_t size, DataType ty) const
+{
+   const char *postFix = "";
+   size_t pos = 0;
+   int idx = join->reg.data.id >= 0 ? join->reg.data.id : id;
+   char p = join->reg.data.id >= 0 ? '$' : '%';
+   char r;
+   int col = TXT_DEFAULT;
+
+   switch (reg.file) {
+   case FILE_GPR:
+      r = 'r'; col = TXT_GPR;
+      if (reg.size == 8)
+         postFix = "d";
+      else
+      if (reg.size == 16)
+         postFix = "q";
+      break;
+   case FILE_PREDICATE:
+      r = 'p'; col = TXT_REGISTER;
+      if (reg.size == 2)
+         postFix = "d";
+      else
+      if (reg.size == 4)
+         postFix = "q";
+      break;
+   case FILE_FLAGS:
+      r = 'c'; col = TXT_FLAGS;
+      break;
+   case FILE_ADDRESS:
+      r = 'a'; col = TXT_REGISTER;
+      break;
+   default:
+      assert(!"invalid file for lvalue");
+      r = '?';
+      break;
+   }
+
+   PRINT("%s%c%c%i%s", colour[col], p, r, idx, postFix);
+
+   return pos;
+}
+
+int ImmediateValue::print(char *buf, size_t size, DataType ty) const
+{
+   size_t pos = 0;
+
+   PRINT("%s", colour[TXT_IMMD]);
+
+   switch (ty) {
+   case TYPE_F32: PRINT("%f", reg.data.f32); break;
+   case TYPE_F64: PRINT("%f", reg.data.f64); break;
+   case TYPE_U8:  PRINT("0x%02x", reg.data.u8); break;
+   case TYPE_S8:  PRINT("%i", reg.data.s8); break;
+   case TYPE_U16: PRINT("0x%04x", reg.data.u16); break;
+   case TYPE_S16: PRINT("%i", reg.data.s16); break;
+   case TYPE_U32: PRINT("0x%08x", reg.data.u32); break;
+   case TYPE_S32: PRINT("%i", reg.data.s32); break;
+   case TYPE_U64:
+   case TYPE_S64:
+   default:
+      PRINT("0x%016lx", reg.data.u64);
+      break;
+   }
+   return pos;
+}
+
+int Symbol::print(char *buf, size_t size, DataType ty) const
+{
+   return print(buf, size, NULL, NULL, ty);
+}
+
+int Symbol::print(char *buf, size_t size,
+                  Value *rel, Value *dimRel, DataType ty) const
+{
+   size_t pos = 0;
+   char c;
+
+   if (ty == TYPE_NONE)
+      ty = typeOfSize(reg.size);
+
+   if (reg.file == FILE_SYSTEM_VALUE) {
+      PRINT("%ssv[%s%s:%i%s", colour[TXT_MEM],
+            colour[TXT_REGISTER],
+            SemanticStr[reg.data.sv.sv], reg.data.sv.index, colour[TXT_MEM]);
+      if (rel) {
+         PRINT("%s+", colour[TXT_DEFAULT]);
+         pos += rel->print(&buf[pos], size - pos);
+      }
+      PRINT("%s]", colour[TXT_MEM]);
+      return pos;
+   }
+
+   switch (reg.file) {
+   case FILE_MEMORY_CONST:  c = 'c'; break;
+   case FILE_SHADER_INPUT:  c = 'a'; break;
+   case FILE_SHADER_OUTPUT: c = 'o'; break;
+   case FILE_MEMORY_GLOBAL: c = 'g'; break;
+   case FILE_MEMORY_SHARED: c = 's'; break;
+   case FILE_MEMORY_LOCAL:  c = 'l'; break;
+   default:
+      assert(!"invalid file");
+      c = '?';
+      break;
+   }
+
+   if (c == 'c')
+      PRINT("%s%c%i[", colour[TXT_MEM], c, reg.fileIndex);
+   else
+      PRINT("%s%c[", colour[TXT_MEM], c);
+
+   if (dimRel) {
+      pos += dimRel->print(&buf[pos], size - pos, TYPE_S32);
+      PRINT("%s][", colour[TXT_MEM]);
+   }
+
+   if (rel) {
+      pos += rel->print(&buf[pos], size - pos);
+      PRINT("%s%c", colour[TXT_DEFAULT], (reg.data.offset < 0) ? '-' : '+');
+   } else {
+      assert(reg.data.offset >= 0);
+   }
+   PRINT("%s0x%x%s]", colour[TXT_IMMD], abs(reg.data.offset), colour[TXT_MEM]);
+
+   return pos;
+}
+
+void Instruction::print() const
+{
+   #define BUFSZ 512
+
+   const size_t size = BUFSZ;
+
+   char buf[BUFSZ];
+   int s, d;
+   size_t pos = 0;
+
+   PRINT("%s", colour[TXT_INSN]);
+
+   if (join)
+      PRINT("join ");
+
+   if (predSrc >= 0) {
+      const size_t pre = pos;
+      if (getSrc(predSrc)->reg.file == FILE_PREDICATE) {
+         if (cc == CC_NOT_P)
+            PRINT("not");
+      } else {
+         PRINT("%s", CondCodeStr[cc]);
+      }
+      if (pos > pre + 1)
+         SPACE();
+      pos += src[predSrc].get()->print(&buf[pos], BUFSZ - pos);
+      PRINT(" %s", colour[TXT_INSN]);
+   }
+
+   if (saturate)
+      PRINT("sat ");
+
+   if (asFlow()) {
+      PRINT("%s", operationStr[op]);
+      if (op == OP_CALL && asFlow()->builtin) {
+         PRINT(" %sBUILTIN:%i", colour[TXT_BRA], asFlow()->target.builtin);
+      } else
+      if (op == OP_CALL && asFlow()->target.fn) {
+         PRINT(" %s%s", colour[TXT_BRA], asFlow()->target.fn->getName());
+      } else
+      if (asFlow()->target.bb)
+         PRINT(" %sBB:%i", colour[TXT_BRA], asFlow()->target.bb->getId());
+   } else {
+      PRINT("%s ", operationStr[op]);
+      if (perPatch)
+         PRINT("patch ");
+      if (asTex())
+         PRINT("%s ", asTex()->tex.target.getName());
+      if (postFactor)
+         PRINT("x2^%i ", postFactor);
+      PRINT("%s%s", dnz ? "dnz " : (ftz ? "ftz " : ""),  DataTypeStr[dType]);
+   }
+
+   if (rnd != ROUND_N)
+      PRINT(" %s", RoundModeStr[rnd]);
+
+   if (def[1].exists())
+      PRINT(" {");
+   for (d = 0; defExists(d); ++d) {
+      SPACE();
+      pos += def[d].get()->print(&buf[pos], size - pos);
+   }
+   if (d > 1)
+      PRINT(" %s}", colour[TXT_INSN]);
+   else
+   if (!d && !asFlow())
+      PRINT(" %s#", colour[TXT_INSN]);
+
+   if (asCmp())
+      PRINT(" %s%s", colour[TXT_INSN], CondCodeStr[asCmp()->setCond]);
+
+   if (sType != dType)
+      PRINT(" %s%s", colour[TXT_INSN], DataTypeStr[sType]);
+
+   for (s = 0; srcExists(s); ++s) {
+      if (s == predSrc || src[s].usedAsPtr)
+         continue;
+      const size_t pre = pos;
+      SPACE();
+      pos += src[s].mod.print(&buf[pos], BUFSZ - pos);
+      if (pos > pre + 1)
+         SPACE();
+      if (src[s].isIndirect(0) || src[s].isIndirect(1))
+         pos += src[s].get()->asSym()->print(&buf[pos], BUFSZ - pos,
+                                             getIndirect(s, 0),
+                                             getIndirect(s, 1));
+      else
+         pos += src[s].get()->print(&buf[pos], BUFSZ - pos, sType);
+   }
+
+   PRINT("%s", colour[TXT_DEFAULT]);
+
+   buf[MIN2(pos, BUFSZ - 1)] = 0;
+
+   INFO("%s (%u)\n", buf, encSize);
+}
+
+class PrintPass : public Pass
+{
+public:
+   PrintPass() : serial(0) { }
+
+   virtual bool visit(Function *);
+   virtual bool visit(BasicBlock *);
+   virtual bool visit(Instruction *);
+
+private:
+   int serial;
+};
+
+bool
+PrintPass::visit(Function *fn)
+{
+   INFO("\n%s:\n", fn->getName());
+
+   return true;
+}
+
+bool
+PrintPass::visit(BasicBlock *bb)
+{
+#if 0
+   INFO("---\n");
+   for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next())
+      INFO(" <- BB:%i (%s)\n",
+           BasicBlock::get(ei.getNode())->getId(),
+           ei.getEdge()->typeStr());
+#endif
+   INFO("BB:%i (%u instructions) - ", bb->getId(), bb->getInsnCount());
+
+   if (bb->idom())
+      INFO("idom = BB:%i, ", bb->idom()->getId());
+
+   INFO("df = { ");
+   for (DLList::Iterator df = bb->getDF().iterator(); !df.end(); df.next())
+      INFO("BB:%i ", BasicBlock::get(df)->getId());
+
+   INFO("}\n");
+
+   for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next())
+      INFO(" -> BB:%i (%s)\n",
+           BasicBlock::get(ei.getNode())->getId(),
+           ei.getEdge()->typeStr());
+
+   return true;
+}
+
+bool
+PrintPass::visit(Instruction *insn)
+{
+   INFO("%3i: ", serial++);
+   insn->print();
+   return true;
+}
+
+void
+Function::print()
+{
+   PrintPass pass;
+   pass.run(this, true, false);
+}
+
+void
+Program::print()
+{
+   PrintPass pass;
+   pass.run(this, true, false);
+}
+
+void
+Function::printLiveIntervals() const
+{
+   INFO("printing live intervals ...\n");
+
+   for (ArrayList::Iterator it = allLValues.iterator(); !it.end(); it.next()) {
+      const Value *lval = Value::get(it)->asLValue();
+      if (lval && !lval->livei.isEmpty()) {
+         INFO("livei(%%%i): ", lval->id);
+         lval->livei.print();
+      }
+   }
+}
+
+} // namespace nv50_ir
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_ra.cpp b/src/gallium/drivers/nv50/codegen/nv50_ir_ra.cpp
new file mode 100644
index 00000000000..7e3c44d3b15
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_ra.cpp
@@ -0,0 +1,963 @@
+
+#include "nv50_ir.h"
+#include "nv50_ir_target.h"
+
+#include "nv50/nv50_debug.h"
+
+namespace nv50_ir {
+
+#define MAX_REGISTER_FILE_SIZE 256
+
+class RegisterSet
+{
+public:
+   RegisterSet();
+   RegisterSet(const Target *);
+
+   void init(const Target *);
+   void reset(); // reset allocation status, but not max assigned regs
+
+   void periodicMask(DataFile f, uint32_t lock, uint32_t unlock);
+   void intersect(DataFile f, const RegisterSet *);
+
+   bool assign(Value **, int nr);
+   void release(const Value *);
+   void occupy(const Value *);
+
+   int getMaxAssigned(DataFile f) const { return fill[f]; }
+
+   void print() const;
+
+private:
+   uint32_t bits[FILE_ADDRESS + 1][(MAX_REGISTER_FILE_SIZE + 31) / 32];
+
+   int unit[FILE_ADDRESS + 1]; // log2 of allocation granularity
+
+   int last[FILE_ADDRESS + 1];
+   int fill[FILE_ADDRESS + 1];
+};
+
+void
+RegisterSet::reset()
+{
+   memset(bits, 0, sizeof(bits));
+}
+
+RegisterSet::RegisterSet()
+{
+   reset();
+}
+
+void
+RegisterSet::init(const Target *targ)
+{
+   for (unsigned int rf = 0; rf <= FILE_ADDRESS; ++rf) {
+      DataFile f = static_cast<DataFile>(rf);
+      last[rf] = targ->getFileSize(f) - 1;
+      unit[rf] = targ->getFileUnit(f);
+      fill[rf] = -1;
+      assert(last[rf] < MAX_REGISTER_FILE_SIZE);
+   }
+}
+
+RegisterSet::RegisterSet(const Target *targ)
+{
+   reset();
+   init(targ);
+}
+
+void
+RegisterSet::periodicMask(DataFile f, uint32_t lock, uint32_t unlock)
+{
+   for (int i = 0; i < (last[f] + 31) / 32; ++i)
+      bits[f][i] = (bits[f][i] | lock) & ~unlock;
+}
+
+void
+RegisterSet::intersect(DataFile f, const RegisterSet *set)
+{
+   for (int i = 0; i < (last[f] + 31) / 32; ++i)
+      bits[f][i] |= set->bits[f][i];
+}
+
+void
+RegisterSet::print() const
+{
+   INFO("GPR:");
+   for (int i = 0; i < (last[FILE_GPR] + 31) / 32; ++i)
+      INFO(" %08x", bits[FILE_GPR][i]);
+   INFO("\n");
+}
+
+bool
+RegisterSet::assign(Value **def, int nr)
+{
+   DataFile f = def[0]->reg.file;
+   int n = nr;
+   if (n == 3)
+      n = 4;
+   int s = (n * def[0]->reg.size) >> unit[f];
+   uint32_t m = (1 << s) - 1;
+
+   int id = last[f] + 1;
+   int i;
+
+   for (i = 0; (i * 32) < last[f]; ++i) {
+      if (bits[f][i] == 0xffffffff)
+         continue;
+
+      for (id = 0; id < 32; id += s)
+         if (!(bits[f][i] & (m << id)))
+            break;
+      if (id < 32)
+         break;
+   }
+   id += i * 32;
+   if (id > last[f])
+      return false;
+
+   bits[f][id / 32] |= m << (id % 32);
+
+   if (id + (s - 1) > fill[f])
+      fill[f] = id + (s - 1);
+
+   for (i = 0; i < nr; ++i, ++id)
+      if (!def[i]->livei.isEmpty()) // XXX: really increased id if empty ?
+         def[i]->reg.data.id = id;
+   return true;
+}
+
+void
+RegisterSet::occupy(const Value *val)
+{
+   int id = val->reg.data.id;
+   if (id < 0)
+      return;
+   unsigned int f = val->reg.file;
+
+   uint32_t m = (1 << (val->reg.size >> unit[f])) - 1;
+
+   INFO_DBG(0, REG_ALLOC, "reg occupy: %u[%i] %x\n", f, id, m);
+
+   bits[f][id / 32] |= m << (id % 32);
+
+   if (fill[f] < id)
+      fill[f] = id;
+}
+
+void
+RegisterSet::release(const Value *val)
+{
+   int id = val->reg.data.id;
+   if (id < 0)
+      return;
+   unsigned int f = val->reg.file;
+
+   uint32_t m = (1 << (val->reg.size >> unit[f])) - 1;
+
+   INFO_DBG(0, REG_ALLOC, "reg release: %u[%i] %x\n", f, id, m);
+
+   bits[f][id / 32] &= ~(m << (id % 32));
+}
+
+#define JOIN_MASK_PHI        (1 << 0)
+#define JOIN_MASK_UNION      (1 << 1)
+#define JOIN_MASK_MOV        (1 << 2)
+#define JOIN_MASK_TEX        (1 << 3)
+#define JOIN_MASK_CONSTRAINT (1 << 4)
+
+class RegAlloc
+{
+public:
+   RegAlloc(Program *program) : prog(program), sequence(0) { }
+
+   bool exec();
+   bool execFunc();
+
+private:
+   bool coalesceValues(unsigned int mask);
+   bool linearScan();
+   bool allocateConstrainedValues();
+
+private:
+   class PhiMovesPass : public Pass {
+   private:
+      virtual bool visit(BasicBlock *);
+      inline bool needNewElseBlock(BasicBlock *b, BasicBlock *p);
+   };
+
+   class BuildIntervalsPass : public Pass {
+   private:
+      virtual bool visit(BasicBlock *);
+      void collectLiveValues(BasicBlock *);
+      void addLiveRange(Value *, const BasicBlock *, int end);
+   };
+
+   class InsertConstraintsPass : public Pass {
+   public:
+      bool exec(Function *func);
+   private:
+      virtual bool visit(BasicBlock *);
+
+      bool insertConstraintMoves();
+
+      void addHazard(Instruction *i, const ValueRef *src);
+      void textureMask(TexInstruction *);
+      void addConstraint(Instruction *, int s, int n);
+      bool detectConflict(Instruction *, int s);
+
+      DLList constrList;
+   };
+
+   bool buildLiveSets(BasicBlock *);
+   void collectLValues(DLList&, bool assignedOnly);
+
+   void insertOrderedTail(DLList&, Value *);
+   inline Instruction *insnBySerial(int);
+
+private:
+   Program *prog;
+   Function *func;
+
+   // instructions in control flow / chronological order
+   ArrayList insns;
+
+   int sequence; // for manual passes through CFG
+};
+
+Instruction *
+RegAlloc::insnBySerial(int serial)
+{
+   return reinterpret_cast<Instruction *>(insns.get(serial));
+}
+
+void
+RegAlloc::BuildIntervalsPass::addLiveRange(Value *val,
+                                           const BasicBlock *bb,
+                                           int end)
+{
+   Instruction *insn = val->getUniqueInsn();
+
+   if (!insn)
+      return;
+   assert(bb->getFirst()->serial <= bb->getExit()->serial);
+   assert(bb->getExit()->serial + 1 >= end);
+
+   int begin = insn->serial;
+   if (begin < bb->getEntry()->serial || begin > bb->getExit()->serial)
+      begin = bb->getEntry()->serial;
+
+   INFO_DBG(prog->dbgFlags, REG_ALLOC, "%%%i <- live range [%i(%i), %i)\n",
+            val->id, begin, insn->serial, end);
+
+   if (begin != end) // empty ranges are only added as hazards for fixed regs
+      val->livei.extend(begin, end);
+}
+
+bool
+RegAlloc::PhiMovesPass::needNewElseBlock(BasicBlock *b, BasicBlock *p)
+{
+   if (b->cfg.incidentCount() <= 1)
+      return false;
+
+   int n = 0;
+   for (Graph::EdgeIterator ei = p->cfg.outgoing(); !ei.end(); ei.next())
+      if (ei.getType() == Graph::Edge::TREE ||
+          ei.getType() == Graph::Edge::FORWARD)
+         ++n;
+   return (n == 2);
+}
+
+// For each operand of each PHI in b, generate a new value by inserting a MOV
+// at the end of the block it is coming from and replace the operand with its
+// result. This eliminates liveness conflicts and enables us to let values be
+// copied to the right register if such a conflict exists nonetheless.
+//
+// These MOVs are also crucial in making sure the live intervals of phi srces
+// are extended until the end of the loop, since they are not included in the
+// live-in sets.
+bool
+RegAlloc::PhiMovesPass::visit(BasicBlock *bb)
+{
+   Instruction *phi, *mov;
+   BasicBlock *pb, *pn;
+
+   for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) {
+      pb = pn = BasicBlock::get(ei.getNode());
+      assert(pb);
+
+      if (needNewElseBlock(bb, pb)) {
+         pn = new BasicBlock(func);
+
+         // deletes an edge, iterator is invalid after this:
+         pb->cfg.detach(&bb->cfg);
+         pb->cfg.attach(&pn->cfg, Graph::Edge::TREE);
+         pn->cfg.attach(&bb->cfg, Graph::Edge::FORWARD); // XXX: check order !
+
+         assert(pb->getExit()->op != OP_CALL);
+         if (pb->getExit()->asFlow()->target.bb == bb)
+            pb->getExit()->asFlow()->target.bb = pn;
+         break;
+      }
+   }
+
+   // insert MOVs (phi->src[j] should stem from j-th in-BB)
+   int j = 0;
+   for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) {
+      pb = BasicBlock::get(ei.getNode());
+      if (!pb->isTerminated())
+         pb->insertTail(new_FlowInstruction(func, OP_BRA, bb));
+
+      for (phi = bb->getPhi(); phi && phi->op == OP_PHI; phi = phi->next) {
+         mov = new_Instruction(func, OP_MOV, TYPE_U32);
+
+         mov->setSrc(0, phi->getSrc(j));
+         mov->setDef(0, new_LValue(func, phi->getDef(0)->asLValue()));
+         phi->setSrc(j, mov->getDef(0));
+
+         pb->insertBefore(pb->getExit(), mov);
+      }
+      ++j;
+   }
+
+   return true;
+}
+
+// Build the set of live-in variables of bb.
+bool
+RegAlloc::buildLiveSets(BasicBlock *bb)
+{
+   BasicBlock *bn;
+   Instruction *i;
+   unsigned int s, d;
+
+   INFO_DBG(prog->dbgFlags, REG_ALLOC, "buildLiveSets(BB:%i)\n", bb->getId());
+
+   bb->liveSet.allocate(func->allLValues.getSize(), false);
+
+   int n = 0;
+   for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
+      bn = BasicBlock::get(ei.getNode());
+      if (bn == bb)
+         continue;
+      if (bn->cfg.visit(sequence))
+         if (!buildLiveSets(bn))
+            return false;
+      if (n++ == 0)
+         bb->liveSet = bn->liveSet;
+      else
+         bb->liveSet |= bn->liveSet;
+   }
+   if (!n && !bb->liveSet.marker)
+      bb->liveSet.fill(0);
+   bb->liveSet.marker = true;
+
+   if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC) {
+      INFO("BB:%i live set of out blocks:\n", bb->getId());
+      bb->liveSet.print();
+   }
+
+   // if (!bb->getEntry())
+   //   return true;
+
+   for (i = bb->getExit(); i && i != bb->getEntry()->prev; i = i->prev) {
+      for (d = 0; i->defExists(d); ++d)
+         bb->liveSet.clr(i->getDef(d)->id);
+      for (s = 0; i->srcExists(s); ++s)
+         if (i->getSrc(s)->asLValue())
+            bb->liveSet.set(i->getSrc(s)->id);
+   }
+   for (i = bb->getPhi(); i && i->op == OP_PHI; i = i->next)
+      bb->liveSet.clr(i->getDef(0)->id);
+
+   if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC) {
+      INFO("BB:%i live set after propagation:\n", bb->getId());
+      bb->liveSet.print();
+   }
+
+   return true;
+}
+
+void
+RegAlloc::BuildIntervalsPass::collectLiveValues(BasicBlock *bb)
+{
+   BasicBlock *bbA = NULL, *bbB = NULL;
+
+   assert(bb->cfg.incidentCount() || bb->liveSet.popCount() == 0);
+
+   if (bb->cfg.outgoingCount()) {
+      // trickery to save a loop of OR'ing liveSets
+      // aliasing works fine with BitSet::setOr
+      for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
+         if (ei.getType() == Graph::Edge::DUMMY)
+            continue;
+         if (bbA) {
+            bb->liveSet.setOr(&bbA->liveSet, &bbB->liveSet);
+            bbA = bb;
+         } else {
+            bbA = bbB;
+         }
+         bbB = BasicBlock::get(ei.getNode());
+      }
+      bb->liveSet.setOr(&bbB->liveSet, bbA ? &bbA->liveSet : NULL);
+   } else
+   if (bb->cfg.incidentCount()) {
+      bb->liveSet.fill(0);
+   }
+}
+
+bool
+RegAlloc::BuildIntervalsPass::visit(BasicBlock *bb)
+{
+   collectLiveValues(bb);
+
+   INFO_DBG(prog->dbgFlags, REG_ALLOC, "BuildIntervals(BB:%i)\n", bb->getId());
+
+   // go through out blocks and delete phi sources that do not originate from
+   // the current block from the live set
+   for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
+      BasicBlock *out = BasicBlock::get(ei.getNode());
+
+      for (Instruction *i = out->getPhi(); i && i->op == OP_PHI; i = i->next) {
+         bb->liveSet.clr(i->getDef(0)->id);
+
+         for (int s = 0; s < NV50_IR_MAX_SRCS && i->src[s].exists(); ++s) {
+            assert(i->src[s].getInsn());
+            if (i->getSrc(s)->getUniqueInsn()->bb == bb) // XXX: reachableBy ?
+               bb->liveSet.set(i->getSrc(s)->id);
+            else
+               bb->liveSet.clr(i->getSrc(s)->id);
+         }
+      }
+   }
+
+   // remaining live-outs are live until end
+   if (bb->getExit()) {
+      for (unsigned int j = 0; j < bb->liveSet.getSize(); ++j)
+         if (bb->liveSet.test(j))
+            addLiveRange(func->getLValue(j), bb, bb->getExit()->serial + 1);
+   }
+
+   for (Instruction *i = bb->getExit(); i && i->op != OP_PHI; i = i->prev) {
+      for (int d = 0; i->defExists(d); ++d) {
+         bb->liveSet.clr(i->getDef(d)->id);
+         if (i->getDef(d)->reg.data.id >= 0) // add hazard for fixed regs
+            i->getDef(d)->livei.extend(i->serial, i->serial);
+      }
+
+      for (int s = 0; i->srcExists(s); ++s) {
+         if (!i->getSrc(s)->asLValue())
+            continue;
+         if (!bb->liveSet.test(i->getSrc(s)->id)) {
+            bb->liveSet.set(i->getSrc(s)->id);
+            addLiveRange(i->getSrc(s), bb, i->serial);
+         }
+      }
+   }
+
+   return true;
+}
+
+bool
+RegAlloc::coalesceValues(unsigned int mask)
+{
+   int c, n;
+
+   for (n = 0; n < insns.getSize(); ++n) {
+      Instruction *i;
+      Instruction *insn = insnBySerial(n);
+
+      switch (insn->op) {
+      case OP_PHI:
+         if (!(mask & JOIN_MASK_PHI))
+            break;
+         for (c = 0; insn->srcExists(c); ++c)
+            if (!insn->getDef(0)->coalesce(insn->getSrc(c), false)) {
+               ERROR("failed to coalesce phi operands\n");
+               return false;
+            }
+         break;
+      case OP_UNION:
+         if (!(mask & JOIN_MASK_UNION))
+            break;
+         for (c = 0; insn->srcExists(c); ++c)
+            insn->getDef(0)->coalesce(insn->getSrc(c), true);
+         break;
+      case OP_CONSTRAINT:
+         if (!(mask & JOIN_MASK_CONSTRAINT))
+            break;
+         for (c = 0; c < 4 && insn->srcExists(c); ++c)
+            insn->getDef(c)->coalesce(insn->getSrc(c), true);
+         break;
+      case OP_MOV:
+         if (!(mask & JOIN_MASK_MOV))
+            break;
+         i = insn->getSrc(0)->getUniqueInsn();
+         if (i && !i->constrainedDefs())
+            insn->getDef(0)->coalesce(insn->getSrc(0), false);
+         break;
+      case OP_TEX:
+      case OP_TXB:
+      case OP_TXL:
+      case OP_TXF:
+      case OP_TXQ:
+      case OP_TXD:
+      case OP_TXG:
+      case OP_TEXCSAA:
+         if (!(mask & JOIN_MASK_TEX))
+            break;
+         for (c = 0; c < 4 && insn->srcExists(c); ++c)
+            insn->getDef(c)->coalesce(insn->getSrc(c), true);
+         break;
+      default:
+         break;
+      }
+   }
+   return true;
+}
+
+void
+RegAlloc::insertOrderedTail(DLList &list, Value *val)
+{
+   // we insert the live intervals in order, so this should be short
+   DLList::Iterator iter = list.revIterator();
+   const int begin = val->livei.begin();
+   for (; !iter.end(); iter.next()) {
+      if (reinterpret_cast<Value *>(iter.get())->livei.begin() <= begin)
+         break;
+   }
+   iter.insert(val);
+}
+
+static void
+checkList(DLList &list)
+{
+   Value *prev = NULL;
+   Value *next = NULL;
+
+   for (DLList::Iterator iter = list.iterator(); !iter.end(); iter.next()) {
+      next = Value::get(iter);
+      assert(next);
+      if (prev) {
+         assert(prev->livei.begin() <= next->livei.begin());
+      }
+      assert(next->join == next);
+      prev = next;
+   }
+}
+
+void
+RegAlloc::collectLValues(DLList &list, bool assignedOnly)
+{
+   for (int n = 0; n < insns.getSize(); ++n) {
+      Instruction *i = insnBySerial(n);
+
+      for (int d = 0; i->defExists(d); ++d)
+         if (!i->getDef(d)->livei.isEmpty())
+            if (!assignedOnly || i->getDef(d)->reg.data.id >= 0)
+               insertOrderedTail(list, i->getDef(d));
+   }
+   checkList(list);
+}
+
+bool
+RegAlloc::allocateConstrainedValues()
+{
+   Value *defs[4];
+   RegisterSet regSet[4];
+   DLList regVals;
+
+   INFO_DBG(prog->dbgFlags, REG_ALLOC, "RA: allocating constrained values\n");
+
+   collectLValues(regVals, true);
+
+   for (int c = 0; c < 4; ++c)
+      regSet[c].init(prog->getTarget());
+
+   for (int n = 0; n < insns.getSize(); ++n) {
+      Instruction *i = insnBySerial(n);
+
+      const int vecSize = i->defCount(0xf);
+      if (vecSize < 2)
+         continue;
+      assert(vecSize <= 4);
+
+      for (int c = 0; c < vecSize; ++c)
+         defs[c] = i->def[c].rep();
+
+      if (defs[0]->reg.data.id >= 0) {
+         for (int c = 1; c < vecSize; ++c) {
+            assert(defs[c]->reg.data.id >= 0);
+         }
+         continue;
+      }
+
+      for (int c = 0; c < vecSize; ++c) {
+         uint32_t mask;
+         regSet[c].reset();
+
+         for (DLList::Iterator it = regVals.iterator(); !it.end(); it.next()) {
+            Value *rVal = Value::get(it);
+            if (rVal->reg.data.id >= 0 && rVal->livei.overlaps(defs[c]->livei))
+               regSet[c].occupy(rVal);
+         }
+         mask = 0x11111111;
+         if (vecSize == 2) // granularity is 2 instead of 4
+            mask |= 0x11111111 << 2;
+         regSet[c].periodicMask(defs[0]->reg.file, 0, ~(mask << c));
+
+         if (!defs[c]->livei.isEmpty())
+            insertOrderedTail(regVals, defs[c]);
+      }
+      for (int c = 1; c < vecSize; ++c)
+         regSet[0].intersect(defs[0]->reg.file, &regSet[c]);
+
+      if (!regSet[0].assign(&defs[0], vecSize)) // TODO: spilling
+         return false;
+   }
+   for (int c = 0; c < 4; c += 2)
+      if (regSet[c].getMaxAssigned(FILE_GPR) > prog->maxGPR)
+         prog->maxGPR = regSet[c].getMaxAssigned(FILE_GPR);
+   return true;
+}
+
+bool
+RegAlloc::linearScan()
+{
+   Value *cur, *val;
+   DLList unhandled, active, inactive;
+   RegisterSet f(prog->getTarget()), free(prog->getTarget());
+
+   INFO_DBG(prog->dbgFlags, REG_ALLOC, "RA: linear scan\n");
+
+   collectLValues(unhandled, false);
+
+   for (DLList::Iterator cI = unhandled.iterator(); !cI.end();) {
+      cur = Value::get(cI);
+      cI.erase();
+
+      for (DLList::Iterator aI = active.iterator(); !aI.end();) {
+         val = Value::get(aI);
+         if (val->livei.end() <= cur->livei.begin()) {
+            free.release(val);
+            aI.erase();
+         } else
+         if (!val->livei.contains(cur->livei.begin())) {
+            free.release(val);
+            aI.moveToList(inactive);
+         } else {
+            aI.next();
+         }
+      }
+
+      for (DLList::Iterator iI = inactive.iterator(); !iI.end();) {
+         val = Value::get(iI);
+         if (val->livei.end() <= cur->livei.begin()) {
+            iI.erase();
+         } else
+         if (val->livei.contains(cur->livei.begin())) {
+            free.occupy(val);
+            iI.moveToList(active);
+         } else {
+            iI.next();
+         }
+      }
+      f = free;
+
+      for (DLList::Iterator iI = inactive.iterator(); !iI.end(); iI.next()) {
+         val = Value::get(iI);
+         if (val->livei.overlaps(cur->livei))
+            f.occupy(val);
+      }
+
+      for (DLList::Iterator uI = unhandled.iterator(); !uI.end(); uI.next()) {
+         val = Value::get(uI);
+         if (val->reg.data.id >= 0 && val->livei.overlaps(cur->livei))
+            f.occupy(val);
+      }
+
+      if (cur->reg.data.id < 0) {
+         bool spill = !f.assign(&cur, 1);
+         if (spill) {
+            ERROR("out of registers of file %u\n", cur->reg.file);
+            abort();
+         }
+      }
+      free.occupy(cur);
+      active.insert(cur);
+   }
+
+   if (f.getMaxAssigned(FILE_GPR) > prog->maxGPR)
+      prog->maxGPR = f.getMaxAssigned(FILE_GPR);
+   if (free.getMaxAssigned(FILE_GPR) > prog->maxGPR)
+      prog->maxGPR = free.getMaxAssigned(FILE_GPR);
+   return true;
+}
+
+bool
+RegAlloc::exec()
+{
+   for (ArrayList::Iterator fi = prog->allFuncs.iterator();
+        !fi.end(); fi.next()) {
+      func = reinterpret_cast<Function *>(fi.get());
+      if (!execFunc())
+         return false;
+   }
+   return true;
+}
+
+bool
+RegAlloc::execFunc()
+{
+   InsertConstraintsPass insertConstr;
+   PhiMovesPass insertMoves;
+   BuildIntervalsPass buildIntervals;
+
+   unsigned int i;
+   bool ret;
+
+   ret = insertConstr.exec(func);
+   if (!ret)
+      goto out;
+
+   ret = insertMoves.run(func);
+   if (!ret)
+      goto out;
+
+   for (sequence = func->cfg.nextSequence(), i = 0;
+        ret && i <= func->loopNestingBound;
+        sequence = func->cfg.nextSequence(), ++i)
+      ret = buildLiveSets(BasicBlock::get(func->cfg.getRoot()));
+   if (!ret)
+      goto out;
+
+   func->orderInstructions(this->insns);
+
+   ret = buildIntervals.run(func);
+   if (!ret)
+      goto out;
+
+   ret = coalesceValues(JOIN_MASK_PHI);
+   if (!ret)
+      goto out;
+   switch (prog->getTarget()->getChipset() & 0xf0) {
+   case 0x50:
+      ret = coalesceValues(JOIN_MASK_UNION | JOIN_MASK_TEX);
+      break;
+   case 0xc0:
+      ret = coalesceValues(JOIN_MASK_UNION | JOIN_MASK_CONSTRAINT);
+      break;
+   default:
+      break;
+   }
+   if (!ret)
+      goto out;
+   ret = coalesceValues(JOIN_MASK_MOV);
+   if (!ret)
+      goto out;
+
+   if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC) {
+      func->print();
+      func->printLiveIntervals();
+   }
+
+   ret = allocateConstrainedValues() && linearScan();
+   if (!ret)
+      goto out;
+
+out:
+   // TODO: should probably call destructor on LValues later instead
+   for (ArrayList::Iterator it = func->allLValues.iterator();
+        !it.end(); it.next())
+      reinterpret_cast<LValue *>(it.get())->livei.clear();
+
+   return ret;
+}
+
+bool Program::registerAllocation()
+{
+   RegAlloc ra(this);
+   return ra.exec();
+}
+
+bool
+RegAlloc::InsertConstraintsPass::exec(Function *ir)
+{
+   constrList.clear();
+
+   bool ret = run(ir, true, true);
+   if (ret)
+      ret = insertConstraintMoves();
+   return ret;
+}
+
+// TODO: make part of texture insn
+void
+RegAlloc::InsertConstraintsPass::textureMask(TexInstruction *tex)
+{
+   Value *def[4];
+   int c, k, d;
+   uint8_t mask = 0;
+
+   for (d = 0, k = 0, c = 0; c < 4; ++c) {
+      if (!(tex->tex.mask & (1 << c)))
+         continue;
+      if (tex->getDef(k)->refCount()) {
+         mask |= 1 << c;
+         def[d++] = tex->getDef(k);
+      }
+      ++k;
+   }
+   tex->tex.mask = mask;
+
+#if 0 // reorder or set the unused ones NULL ?
+   for (c = 0; c < 4; ++c)
+      if (!(tex->tex.mask & (1 << c)))
+         def[d++] = tex->getDef(c);
+#endif
+   for (c = 0; c < d; ++c)
+      tex->setDef(c, def[c]);
+#if 1
+   for (; c < 4; ++c)
+      tex->setDef(c, NULL);
+#endif
+}
+
+bool
+RegAlloc::InsertConstraintsPass::detectConflict(Instruction *cst, int s)
+{
+   // current register allocation can't handle it if a value participates in
+   // multiple constraints
+   for (ValueRef::Iterator it = cst->src[s].iterator(); !it.end(); it.next()) {
+      Instruction *insn = it.get()->getInsn();
+      if (insn != cst)
+         return true;
+   }
+
+   // can start at s + 1 because detectConflict is called on all sources
+   for (int c = s + 1; cst->srcExists(c); ++c)
+      if (cst->getSrc(c) == cst->getSrc(s))
+         return true;
+
+   Instruction *defi = cst->getSrc(s)->getInsn();
+
+   return (!defi || defi->constrainedDefs());
+}
+
+void
+RegAlloc::InsertConstraintsPass::addConstraint(Instruction *i, int s, int n)
+{
+   Instruction *cst;
+   int d;
+
+   // first, look for an existing identical constraint op
+   for (DLList::Iterator it = constrList.iterator(); !it.end(); it.next()) {
+      cst = reinterpret_cast<Instruction *>(it.get());
+      if (!i->bb->dominatedBy(cst->bb))
+         break;
+      for (d = 0; d < n; ++d)
+         if (cst->getSrc(d) != i->getSrc(d + s))
+            break;
+      if (d >= n) {
+         for (d = 0; d < n; ++d, ++s)
+            i->setSrc(s, cst->getDef(d));
+         return;
+      }
+   }
+   cst = new_Instruction(func, OP_CONSTRAINT, i->dType);
+
+   for (d = 0; d < n; ++s, ++d) {
+      cst->setDef(d, new_LValue(func, FILE_GPR));
+      cst->setSrc(d, i->getSrc(s));
+      i->setSrc(s, cst->getDef(d));
+   }
+   i->bb->insertBefore(i, cst);
+
+   constrList.insert(cst);
+}
+
+// Add a dummy use of the pointer source of >= 8 byte loads after the load
+// to prevent it from being assigned a register which overlapping the load's
+// destination, which would produce random corruptions.
+void
+RegAlloc::InsertConstraintsPass::addHazard(Instruction *i, const ValueRef *src)
+{
+   Instruction *hzd = new_Instruction(func, OP_NOP, TYPE_NONE);
+   hzd->setSrc(0, src->get());
+   i->bb->insertAfter(i, hzd);
+
+}
+
+// Insert constraint markers for instructions whose multiple sources must be
+// located in consecutive registers.
+bool
+RegAlloc::InsertConstraintsPass::visit(BasicBlock *bb)
+{
+   TexInstruction *tex;
+   Instruction *next;
+   int s, n, size;
+
+   for (Instruction *i = bb->getEntry(); i; i = next) {
+      next = i->next;
+
+      if ((tex = i->asTex())) {
+         textureMask(tex);
+
+         // FIXME: this is target specific
+         if (tex->op == OP_TXQ) {
+            s = tex->srcCount(0xff);
+            n = 0;
+         } else {
+            s = tex->tex.target.getArgCount();
+            if (!tex->tex.target.isArray() &&
+                (tex->tex.rIndirectSrc >= 0 || tex->tex.sIndirectSrc >= 0))
+               ++s;
+            n = tex->srcCount(0xff) - s;
+            assert(n <= 4);
+         }
+
+         if (s > 1)
+            addConstraint(i, 0, s);
+         if (n > 1)
+            addConstraint(i, s, n);
+      } else
+      if (i->op == OP_EXPORT || i->op == OP_STORE) {
+         for (size = typeSizeof(i->dType), s = 1; size > 0; ++s) {
+            assert(i->srcExists(s));
+            size -= i->getSrc(s)->reg.size;
+         }
+         if ((s - 1) > 1)
+            addConstraint(i, 1, s - 1);
+      } else
+      if (i->op == OP_LOAD) {
+         if (i->src[0].isIndirect(0) && typeSizeof(i->dType) >= 8)
+            addHazard(i, i->src[0].getIndirect(0));
+      }
+   }
+   return true;
+}
+
+// Insert extra moves so that, if multiple register constraints on a value are
+// in conflict, these conflicts can be resolved.
+bool
+RegAlloc::InsertConstraintsPass::insertConstraintMoves()
+{
+   for (DLList::Iterator it = constrList.iterator(); !it.end(); it.next()) {
+      Instruction *cst = reinterpret_cast<Instruction *>(it.get());
+
+      for (int s = 0; cst->srcExists(s); ++s) {
+         if (!detectConflict(cst, s))
+             continue;
+         Instruction *mov = new_Instruction(func, OP_MOV,
+                                            typeOfSize(cst->src[s].getSize()));
+         mov->setSrc(0, cst->getSrc(s));
+         mov->setDef(0, new_LValue(func, FILE_GPR));
+         cst->setSrc(s, mov->getDef(0));
+
+         cst->bb->insertBefore(cst, mov);
+      }
+   }
+   return true;
+}
+
+} // namespace nv50_ir
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_ssa.cpp b/src/gallium/drivers/nv50/codegen/nv50_ir_ssa.cpp
new file mode 100644
index 00000000000..841163b0ac9
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_ssa.cpp
@@ -0,0 +1,463 @@
+
+#include "nv50_ir.h"
+#include "nv50_ir_target.h"
+
+namespace nv50_ir {
+
+// Converts nv50 IR generated from TGSI to SSA form.
+
+// DominatorTree implements an algorithm for finding immediate dominators,
+// as described by T. Lengauer & R. Tarjan.
+class DominatorTree : public Graph
+{
+public:
+   DominatorTree(Graph *cfg);
+   ~DominatorTree() { }
+   
+   bool dominates(BasicBlock *, BasicBlock *);
+
+   void findDominanceFrontiers();
+
+private:
+   void build();
+   void buildDFS(Node *);
+
+   void squash(int);
+   inline void link(int, int);
+   inline int eval(int);
+
+   void debugPrint();
+
+   Graph *cfg;
+
+   Node **vert;
+   int *data;
+   const int count;
+
+   #define SEMI(i)     (data[(i) + 0 * count])
+   #define ANCESTOR(i) (data[(i) + 1 * count])
+   #define PARENT(i)   (data[(i) + 2 * count])
+   #define LABEL(i)    (data[(i) + 3 * count])
+   #define DOM(i)      (data[(i) + 4 * count])
+};
+
+void DominatorTree::debugPrint()
+{
+   for (int i = 0; i < count; ++i) {
+      INFO("SEMI(%i) = %i\n", i, SEMI(i));
+      INFO("ANCESTOR(%i) = %i\n", i, ANCESTOR(i));
+      INFO("PARENT(%i) = %i\n", i, PARENT(i));
+      INFO("LABEL(%i) = %i\n", i, LABEL(i));
+      INFO("DOM(%i) = %i\n", i, DOM(i));
+   }
+}
+
+DominatorTree::DominatorTree(Graph *cfgraph) : cfg(cfgraph),
+                                               count(cfg->getSize())
+{
+   Iterator *iter;
+   int i;
+
+   vert = new Node * [count];
+   data = new int[5 * count];
+
+   for (i = 0, iter = cfg->iteratorDFS(true); !iter->end(); iter->next(), ++i) {
+      vert[i] = reinterpret_cast<Node *>(iter->get());
+      vert[i]->tag = i;
+      LABEL(i) = i;
+      SEMI(i) = ANCESTOR(i) = -1;
+   }
+   cfg->putIterator(iter);
+
+   build();
+
+   delete[] vert;
+   delete[] data;
+}
+
+void DominatorTree::buildDFS(Graph::Node *node)
+{
+   SEMI(node->tag) = node->tag;
+
+   for (Graph::EdgeIterator ei = node->outgoing(); !ei.end(); ei.next()) {
+      if (SEMI(ei.getNode()->tag) < 0) {
+         buildDFS(ei.getNode());
+         PARENT(ei.getNode()->tag) = node->tag;
+      }
+   }
+}
+
+void DominatorTree::squash(int v)
+{
+   if (ANCESTOR(ANCESTOR(v)) >= 0) {
+      squash(ANCESTOR(v));
+
+      if (SEMI(LABEL(ANCESTOR(v))) < SEMI(LABEL(v)))
+         LABEL(v) = LABEL(ANCESTOR(v));
+      ANCESTOR(v) = ANCESTOR(ANCESTOR(v));
+   }
+}
+
+int DominatorTree::eval(int v)
+{
+   if (ANCESTOR(v) < 0)
+      return v;
+   squash(v);
+   return LABEL(v);
+}
+
+void DominatorTree::link(int v, int w)
+{
+   ANCESTOR(w) = v;
+}
+
+void DominatorTree::build()
+{
+   DLList *bucket = new DLList[count];
+   Node *nv, *nw;
+   int p, u, v, w;
+
+   buildDFS(cfg->getRoot());
+
+   for (w = count - 1; w >= 1; --w) {
+      nw = vert[w];
+      assert(nw->tag == w);
+      for (Graph::EdgeIterator ei = nw->incident(); !ei.end(); ei.next()) {
+         nv = ei.getNode();
+         v = nv->tag;
+         u = eval(v);
+         if (SEMI(u) < SEMI(w))
+            SEMI(w) = SEMI(u);
+      }
+      p = PARENT(w);
+      bucket[SEMI(w)].insert(nw);
+      link(p, w);
+
+      for (DLList::Iterator it = bucket[p].iterator(); !it.end(); it.erase()) {
+         v = reinterpret_cast<Node *>(it.get())->tag;
+         u = eval(v);
+         DOM(v) = (SEMI(u) < SEMI(v)) ? u : p;
+      }
+   }
+   for (w = 1; w < count; ++w) {
+      if (DOM(w) != SEMI(w))
+         DOM(w) = DOM(DOM(w));
+   }
+   DOM(0) = 0;
+
+   insert(&BasicBlock::get(cfg->getRoot())->dom);
+   do {
+      p = 0;
+      for (v = 1; v < count; ++v) {
+         nw = &BasicBlock::get(vert[DOM(v)])->dom;;
+         nv = &BasicBlock::get(vert[v])->dom;
+         if (nw->getGraph() && !nv->getGraph()) {
+            ++p;
+            nw->attach(nv, Graph::Edge::TREE);
+         }
+      }
+   } while (p);
+
+   delete[] bucket;
+}
+
+#undef SEMI
+#undef ANCESTOR
+#undef PARENT
+#undef LABEL
+#undef DOM
+
+void DominatorTree::findDominanceFrontiers()
+{
+   Iterator *dtIter;
+   BasicBlock *bb;
+
+   for (dtIter = this->iteratorDFS(false); !dtIter->end(); dtIter->next()) {
+      EdgeIterator succIter, chldIter;
+
+      bb = BasicBlock::get(reinterpret_cast<Node *>(dtIter->get()));
+      bb->getDF().clear();
+
+      for (succIter = bb->cfg.outgoing(); !succIter.end(); succIter.next()) {
+         BasicBlock *dfLocal = BasicBlock::get(succIter.getNode());
+         if (dfLocal->idom() != bb)
+            bb->getDF().insert(dfLocal);
+      }
+
+      for (chldIter = bb->dom.outgoing(); !chldIter.end(); chldIter.next()) {
+         BasicBlock *cb = BasicBlock::get(chldIter.getNode());
+
+         DLList::Iterator dfIter = cb->getDF().iterator();
+         for (; !dfIter.end(); dfIter.next()) {
+            BasicBlock *dfUp = BasicBlock::get(dfIter);
+            if (dfUp->idom() != bb)
+               bb->getDF().insert(dfUp);
+         }
+      }
+   }
+   this->putIterator(dtIter);
+}
+
+// liveIn(bb) = usedBeforeAssigned(bb) U (liveOut(bb) - assigned(bb))
+void
+Function::buildLiveSetsPreSSA(BasicBlock *bb, const int seq)
+{
+   BitSet usedBeforeAssigned(allLValues.getSize(), true);
+   BitSet assigned(allLValues.getSize(), true);
+
+   bb->liveSet.allocate(allLValues.getSize(), false);
+
+   int n = 0;
+   for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
+      BasicBlock *out = BasicBlock::get(ei.getNode());
+      if (out == bb)
+         continue;
+      if (out->cfg.visit(seq))
+         buildLiveSetsPreSSA(out, seq);
+      if (!n++)
+         bb->liveSet = out->liveSet;
+      else
+         bb->liveSet |= out->liveSet;
+   }
+   if (!n && !bb->liveSet.marker)
+      bb->liveSet.fill(0);
+   bb->liveSet.marker = true;
+
+   for (Instruction *i = bb->getEntry(); i; i = i->next) {
+      for (int s = 0; i->srcExists(s); ++s)
+         if (i->getSrc(s)->asLValue() && !assigned.test(i->getSrc(s)->id))
+            usedBeforeAssigned.set(i->getSrc(s)->id);
+      for (int d = 0; i->defExists(d); ++d)
+         assigned.set(i->getDef(d)->id);
+   }
+
+   bb->liveSet.andNot(assigned);
+   bb->liveSet |= usedBeforeAssigned;
+}
+
+class RenamePass
+{
+public:
+   RenamePass(Function *);
+   ~RenamePass();
+
+   bool run();
+   void search(BasicBlock *);
+
+   inline LValue *getStackTop(Value *);
+
+private:
+   Stack *stack;
+   Function *func;
+   Program *prog;
+   Instruction *undef;
+};
+
+bool
+Program::convertToSSA()
+{
+   for (ArrayList::Iterator fi = allFuncs.iterator(); !fi.end(); fi.next()) {
+      Function *fn = reinterpret_cast<Function *>(fi.get());
+      if (!fn->convertToSSA())
+         return false;
+   }
+   return true;
+}
+
+// XXX: add edge from entry to exit ?
+
+// Efficiently Computing Static Single Assignment Form and
+//  the Control Dependence Graph,
+// R. Cytron, J. Ferrante, B. K. Rosen, M. N. Wegman, F. K. Zadeck
+bool
+Function::convertToSSA()
+{
+   // 0. calculate live in variables (for pruned SSA)
+   int seq = cfg.nextSequence();
+   for (unsigned i = 0; i <= loopNestingBound; seq = cfg.nextSequence(), ++i)
+      buildLiveSetsPreSSA(BasicBlock::get(cfg.getRoot()), seq);
+
+   // reset liveSet marker for use in regalloc
+   for (ArrayList::Iterator bi = allBBlocks.iterator(); !bi.end(); bi.next())
+      reinterpret_cast<BasicBlock *>(bi.get())->liveSet.marker = false;
+
+   // 1. create the dominator tree
+   domTree = new DominatorTree(&cfg);
+   reinterpret_cast<DominatorTree *>(domTree)->findDominanceFrontiers();
+
+   // 2. insert PHI functions
+   DLList workList;
+   LValue *lval;
+   BasicBlock *bb;
+   int var;
+   int iterCount = 0;
+   int *hasAlready = new int[allBBlocks.getSize() * 2];
+   int *work = &hasAlready[allBBlocks.getSize()];
+
+   memset(hasAlready, 0, allBBlocks.getSize() * 2 * sizeof(int));
+
+   // for each variable
+   for (var = 0; var < allLValues.getSize(); ++var) {
+      if (!allLValues.get(var))
+         continue;
+      lval = reinterpret_cast<Value *>(allLValues.get(var))->asLValue();
+      if (!lval || !lval->defs)
+         continue;
+      ++iterCount;
+
+      // TODO: don't add phi functions for values that aren't used outside
+      //  the BB they're defined in
+
+      // gather blocks with assignments to lval in workList
+      for (ValueDef::Iterator d = lval->defs->iterator(); !d.end(); d.next()) {
+         bb = d.get()->getInsn()->bb;
+         if (!bb)
+            continue; // instruction likely been removed but not XXX deleted
+
+         if (work[bb->getId()] == iterCount)
+            continue;
+         work[bb->getId()] = iterCount;
+         workList.insert(bb);
+      }
+
+      // for each block in workList, insert a phi for lval in the block's
+      //  dominance frontier (if we haven't already done so)
+      for (DLList::Iterator wI = workList.iterator(); !wI.end(); wI.erase()) {
+         bb = BasicBlock::get(wI);
+
+         DLList::Iterator dfIter = bb->getDF().iterator();
+         for (; !dfIter.end(); dfIter.next()) {
+            Instruction *phi;
+            BasicBlock *dfBB = BasicBlock::get(dfIter);
+
+            if (hasAlready[dfBB->getId()] >= iterCount)
+               continue;
+            hasAlready[dfBB->getId()] = iterCount;
+
+            // pruned SSA: don't need a phi if the value is not live-in
+            if (!dfBB->liveSet.test(lval->id))
+               continue;
+
+            // TODO: use dedicated PhiInstruction to lift this limit
+            assert(dfBB->cfg.incidentCount() <= NV50_IR_MAX_SRCS);
+
+            phi = new_Instruction(this, OP_PHI, typeOfSize(lval->reg.size));
+            dfBB->insertTail(phi);
+
+            phi->setDef(0, lval);
+            for (int s = 0; s < dfBB->cfg.incidentCount(); ++s)
+               phi->setSrc(s, lval);
+
+            if (work[dfBB->getId()] < iterCount) {
+               work[dfBB->getId()] = iterCount;
+               wI.insert(dfBB);
+            }
+         }
+      }
+   }
+   delete[] hasAlready;
+
+   RenamePass rename(this);
+   return rename.run();
+}
+
+RenamePass::RenamePass(Function *fn) : func(fn), prog(fn->getProgram())
+{
+   BasicBlock *root = BasicBlock::get(func->cfg.getRoot());
+
+   undef = new_Instruction(func, OP_NOP, TYPE_U32);
+   undef->setDef(0, new_LValue(func, FILE_GPR));
+   root->insertHead(undef);
+
+   stack = new Stack[func->allLValues.getSize()];
+}
+
+RenamePass::~RenamePass()
+{
+   if (stack)
+      delete[] stack;
+}
+
+LValue *
+RenamePass::getStackTop(Value *val)
+{
+   if (!stack[val->id].getSize())
+      return 0;
+   return reinterpret_cast<LValue *>(stack[val->id].peek().u.p);
+}
+
+bool RenamePass::run()
+{
+   if (!stack)
+      return false;
+   search(BasicBlock::get(func->domTree->getRoot()));
+
+   ArrayList::Iterator iter = func->allInsns.iterator();
+   for (; !iter.end(); iter.next()) {
+      Instruction *insn = reinterpret_cast<Instruction *>(iter.get());
+      for (int d = 0; insn->defExists(d); ++d)
+         insn->def[d].restoreDefList();
+   }
+
+   return true;
+}
+
+void RenamePass::search(BasicBlock *bb)
+{
+   LValue *lval;
+   int d, s;
+   const Target *targ = prog->getTarget();
+
+   for (Instruction *stmt = bb->getFirst(); stmt; stmt = stmt->next) {
+      if (stmt->op != OP_PHI) {
+         for (s = 0; stmt->srcExists(s); ++s) {
+            lval = stmt->getSrc(s)->asLValue();
+            if (!lval)
+               continue;
+            lval = getStackTop(lval);
+            if (!lval)
+               lval = static_cast<LValue *>(undef->getDef(0));
+            stmt->setSrc(s, lval);
+         }
+      }
+      for (d = 0; stmt->defExists(d); ++d) {
+         lval = stmt->def[d].get()->asLValue();
+         assert(lval);
+         stmt->def[d].setSSA(
+            new_LValue(func, targ->nativeFile(lval->reg.file)));
+         stmt->def[d].get()->reg.data.id = lval->reg.data.id;
+         stack[lval->id].push(stmt->def[d].get());
+      }
+   }
+
+   for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
+      Instruction *phi;
+      int p = 0;
+      BasicBlock *sb = BasicBlock::get(ei.getNode());
+
+      // which predecessor of sb is bb ?
+      for (Graph::EdgeIterator ei = sb->cfg.incident(); !ei.end(); ei.next()) {
+         if (ei.getNode() == &bb->cfg)
+            break;
+         ++p;
+      }
+      assert(p < sb->cfg.incidentCount());
+
+      for (phi = sb->getPhi(); phi && phi->op == OP_PHI; phi = phi->next) {
+         lval = getStackTop(phi->getSrc(p));
+         if (!lval)
+            lval = undef->getDef(0)->asLValue();
+         phi->setSrc(p, lval);
+      }
+   }
+
+   for (Graph::EdgeIterator ei = bb->dom.outgoing(); !ei.end(); ei.next())
+      search(BasicBlock::get(ei.getNode()));
+
+   for (Instruction *stmt = bb->getFirst(); stmt; stmt = stmt->next) {
+      for (d = 0; stmt->defExists(d); ++d)
+         stack[stmt->def[d].preSSA()->id].pop();
+   }
+}
+
+} // namespace nv50_ir
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_target.cpp b/src/gallium/drivers/nv50/codegen/nv50_ir_target.cpp
new file mode 100644
index 00000000000..59fb0c19b0b
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_target.cpp
@@ -0,0 +1,304 @@
+
+#include "nv50/codegen/nv50_ir.h"
+#include "nv50/codegen/nv50_ir_target.h"
+
+namespace nv50_ir {
+
+const uint8_t Target::operationSrcNr[OP_LAST + 1] =
+{
+   0, 0,                   // NOP, PHI
+   0, 0, 0, 0,             // UNION, SPLIT, MERGE, CONSTRAINT
+   1, 1, 2,                // MOV, LOAD, STORE
+   2, 2, 2, 2, 2, 3, 3, 3, // ADD, SUB, MUL, DIV, MOD, MAD, FMA, SAD
+   1, 1, 1,                // ABS, NEG, NOT
+   2, 2, 2, 2, 2,          // AND, OR, XOR, SHL, SHR
+   2, 2, 1,                // MAX, MIN, SAT
+   1, 1, 1, 1,             // CEIL, FLOOR, TRUNC, CVT
+   3, 3, 3, 2, 3, 3,       // SET_AND,OR,XOR, SET, SELP, SLCT
+   1, 1, 1, 1, 1, 1,       // RCP, RSQ, LG2, SIN, COS, EX2
+   1, 1, 1, 1, 1, 2,       // EXP, LOG, PRESIN, PREEX2, SQRT, POW
+   0, 0, 0, 0, 0,          // BRA, CALL, RET, CONT, BREAK,
+   0, 0, 0,                // PRERET,CONT,BREAK
+   0, 0, 0, 0, 0, 0,       // BRKPT, JOINAT, JOIN, DISCARD, EXIT, MEMBAR
+   1, 1, 2, 1, 2,          // VFETCH, PFETCH, EXPORT, LINTERP, PINTERP
+   1, 1,                   // EMIT, RESTART
+   1, 1, 1,                // TEX, TXB, TXL,
+   1, 1, 1, 1, 1,          // TXF, TXQ, TXD, TXG, TEXCSAA
+   1, 2,                   // SULD, SUST
+   1, 1,                   // DFDX, DFDY
+   1, 2, 2, 2, 0, 0,       // RDSV, WRSV, PIXLD, QUADOP, QUADON, QUADPOP
+   2, 3, 2,                // POPCNT, INSBF, EXTBF
+   0
+};
+
+
+extern Target *getTargetNVC0(unsigned int chipset);
+
+Target *Target::create(unsigned int chipset)
+{
+   switch (chipset & 0xf0) {
+   case 0xc0:
+      return getTargetNVC0(chipset);
+   case 0x50:
+   case 0x80:
+   case 0x90:
+   case 0xa0:
+   default:
+      ERROR("unsupported target: NV%x\n", chipset);
+      return 0;
+   }
+}
+
+void Target::destroy(Target *targ)
+{
+   delete targ;
+}
+
+void
+CodeEmitter::setCodeLocation(void *ptr, uint32_t size)
+{
+   code = reinterpret_cast<uint32_t *>(ptr);
+   codeSize = 0;
+   codeSizeLimit = size;
+}
+
+void
+CodeEmitter::printBinary() const
+{
+   uint32_t *bin = code - codeSize / 4;
+   INFO("program binary (%u bytes)", codeSize);
+   for (unsigned int pos = 0; pos < codeSize / 4; ++pos) {
+      if ((pos % 8) == 0)
+         INFO("\n");
+      INFO("%08x ", bin[pos]);
+   }
+   INFO("\n");
+}
+
+void
+CodeEmitter::prepareEmission(Program *prog)
+{
+   for (ArrayList::Iterator fi = prog->allFuncs.iterator();
+        !fi.end(); fi.next()) {
+      Function *func = reinterpret_cast<Function *>(fi.get());
+      func->binPos = prog->binSize;
+      prepareEmission(func);
+      prog->binSize += func->binSize;
+   }
+}
+
+void
+CodeEmitter::prepareEmission(Function *func)
+{
+   func->bbCount = 0;
+   func->bbArray = new BasicBlock * [func->cfg.getSize()];
+
+   BasicBlock::get(func->cfg.getRoot())->binPos = func->binPos;
+
+   Graph::GraphIterator *iter;
+   for (iter = func->cfg.iteratorCFG(); !iter->end(); iter->next())
+      prepareEmission(BasicBlock::get(*iter));
+   func->cfg.putIterator(iter);
+}
+
+void
+CodeEmitter::prepareEmission(BasicBlock *bb)
+{
+   Instruction *i, *next;
+   Function *func = bb->getFunction();
+   int j;
+   unsigned int nShort;
+
+   for (j = func->bbCount - 1; j >= 0 && !func->bbArray[j]->binSize; --j);
+
+   for (; j >= 0; --j) {
+      BasicBlock *in = func->bbArray[j];
+      Instruction *exit = in->getExit();
+
+      if (exit && exit->op == OP_BRA && exit->asFlow()->target.bb == bb) {
+         in->binSize -= 8;
+         func->binSize -= 8;
+
+         for (++j; j < func->bbCount; ++j)
+            func->bbArray[j]->binPos -= 8;
+
+         in->remove(exit);
+      }
+      bb->binPos = in->binPos + in->binSize;
+      if (in->binSize) // no more no-op branches to bb
+         break;
+   }
+   func->bbArray[func->bbCount++] = bb;
+
+   if (!bb->getExit())
+      return;
+
+   // determine encoding size, try to group short instructions
+   nShort = 0;
+   for (i = bb->getEntry(); i; i = next) {
+      next = i->next;
+
+      i->encSize = getMinEncodingSize(i);
+      if (next && i->encSize < 8)
+         ++nShort;
+      else
+      if ((nShort & 1) && next && getMinEncodingSize(next) == 4) {
+         if (i->isCommutationLegal(i->next)) {
+            bb->permuteAdjacent(i, next);
+            next->encSize = 4;
+            next = i;
+            i = i->prev;
+            ++nShort;
+         } else
+         if (i->isCommutationLegal(i->prev) && next->next) {
+            bb->permuteAdjacent(i->prev, i);
+            next->encSize = 4;
+            next = next->next;
+            bb->binSize += 4;
+            ++nShort;
+         } else {
+            i->encSize = 8;
+            i->prev->encSize = 8;
+            bb->binSize += 4;
+            nShort = 0;
+         }
+      } else {
+         i->encSize = 8;
+         if (nShort & 1) {
+            i->prev->encSize = 8;
+            bb->binSize += 4;
+         }
+         nShort = 0;
+      }
+      bb->binSize += i->encSize;
+   }
+
+   if (bb->getExit()->encSize == 4) {
+      assert(nShort);
+      bb->getExit()->encSize = 8;
+      bb->binSize += 4;
+
+      if ((bb->getExit()->prev->encSize == 4) && !(nShort & 1)) {
+         bb->binSize += 8;
+         bb->getExit()->prev->encSize = 8;
+      }
+   }
+   assert(!bb->getEntry() || (bb->getExit() && bb->getExit()->encSize == 8));
+
+   func->binSize += bb->binSize;
+}
+
+bool
+Program::emitBinary(struct nv50_ir_prog_info *info)
+{
+   CodeEmitter *emit = target->getCodeEmitter(progType);
+
+   emit->prepareEmission(this);
+
+   if (dbgFlags & NV50_IR_DEBUG_BASIC)
+      this->print();
+
+   if (!binSize) {
+      code = NULL;
+      return false;
+   }
+   code = reinterpret_cast<uint32_t *>(MALLOC(binSize));
+   if (!code)
+      return false;
+   emit->setCodeLocation(code, binSize);
+
+   for (ArrayList::Iterator fi = allFuncs.iterator(); !fi.end(); fi.next()) {
+      Function *fn = reinterpret_cast<Function *>(fi.get());
+
+      assert(emit->getCodeSize() == fn->binPos);
+
+      for (int b = 0; b < fn->bbCount; ++b)
+         for (Instruction *i = fn->bbArray[b]->getEntry(); i; i = i->next)
+            emit->emitInstruction(i);
+   }
+   info->bin.relocData = emit->getRelocInfo();
+
+   delete emit;
+   return true;
+}
+
+#define RELOC_ALLOC_INCREMENT 8
+
+bool
+CodeEmitter::addReloc(RelocEntry::Type ty, int w, uint32_t data, uint32_t m,
+                      int s)
+{
+   unsigned int n = relocInfo ? relocInfo->count : 0;
+
+   if (!(n % RELOC_ALLOC_INCREMENT)) {
+      size_t size = sizeof(RelocInfo) + n * sizeof(RelocEntry);
+      relocInfo = reinterpret_cast<RelocInfo *>(
+         REALLOC(relocInfo, n ? size : 0,
+                 size + RELOC_ALLOC_INCREMENT * sizeof(RelocEntry)));
+      if (!relocInfo)
+         return false;
+   }
+   ++relocInfo->count;
+
+   relocInfo->entry[n].data = data;
+   relocInfo->entry[n].mask = m;
+   relocInfo->entry[n].offset = codeSize + w * 4;
+   relocInfo->entry[n].bitPos = s;
+   relocInfo->entry[n].type = ty;
+
+   return true;
+}
+
+void
+RelocEntry::apply(uint32_t *binary, const RelocInfo *info) const
+{
+   uint32_t value = 0;
+
+   switch (type) {
+   case TYPE_CODE: value = info->codePos; break;
+   case TYPE_BUILTIN: value = info->libPos; break;
+   case TYPE_DATA: value = info->dataPos; break;
+   default:
+      assert(0);
+      break;
+   }
+   value += data;
+   value = (bitPos < 0) ? (value >> -bitPos) : (value << bitPos);
+
+   binary[offset / 4] &= ~mask;
+   binary[offset / 4] |= value & mask;
+}
+
+} // namespace nv50_ir
+
+
+#include "nv50/codegen/nv50_ir_driver.h"
+
+extern "C" {
+
+void
+nv50_ir_relocate_code(void *relocData, uint32_t *code,
+                      uint32_t codePos,
+                      uint32_t libPos,
+                      uint32_t dataPos)
+{
+   nv50_ir::RelocInfo *info = reinterpret_cast<nv50_ir::RelocInfo *>(relocData);
+
+   info->codePos = codePos;
+   info->libPos = libPos;
+   info->dataPos = dataPos;
+
+   for (unsigned int i = 0; i < info->count; ++i)
+      info->entry[i].apply(code, info);
+}
+
+void
+nv50_ir_get_target_library(uint32_t chipset,
+                           const uint32_t **code, uint32_t *size)
+{
+   nv50_ir::Target *targ = nv50_ir::Target::create(chipset);
+   targ->getBuiltinCode(code, size);
+   nv50_ir::Target::destroy(targ);
+}
+
+}
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_target.h b/src/gallium/drivers/nv50/codegen/nv50_ir_target.h
new file mode 100644
index 00000000000..ddde5586890
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_target.h
@@ -0,0 +1,164 @@
+
+#ifndef __NV50_IR_TARGET_H__
+#define __NV50_IR_TARGET_H__
+
+#include "nv50_ir.h"
+
+namespace nv50_ir {
+
+struct RelocInfo;
+
+struct RelocEntry
+{
+   enum Type
+   {
+      TYPE_CODE,
+      TYPE_BUILTIN,
+      TYPE_DATA
+   };
+
+   uint32_t data;
+   uint32_t mask;
+   uint32_t offset;
+   int8_t bitPos;
+   Type type;
+
+   inline void apply(uint32_t *binary, const RelocInfo *info) const;
+};
+
+struct RelocInfo
+{
+   uint32_t codePos;
+   uint32_t libPos;
+   uint32_t dataPos;
+
+   uint32_t count;
+
+   RelocEntry entry[0];
+};
+
+class CodeEmitter
+{
+public:
+   // returns whether the instruction was encodable and written
+   virtual bool emitInstruction(Instruction *) = 0;
+
+   virtual uint32_t getMinEncodingSize(const Instruction *) const = 0;
+
+   void setCodeLocation(void *, uint32_t size);
+   inline void *getCodeLocation() const { return code; }
+   inline uint32_t getCodeSize() const { return codeSize; }
+
+   bool addReloc(RelocEntry::Type, int w, uint32_t data, uint32_t m,
+                 int s);
+
+   inline void *getRelocInfo() const { return relocInfo; }
+
+   void prepareEmission(Program *);
+   void prepareEmission(Function *);
+   virtual void prepareEmission(BasicBlock *);
+
+   void printBinary() const;
+
+protected:
+   uint32_t *code;
+   uint32_t codeSize;
+   uint32_t codeSizeLimit;
+
+   RelocInfo *relocInfo;
+};
+
+class Target
+{
+public:
+   static Target *create(uint32_t chipset);
+   static void destroy(Target *);
+
+   // 0x50 and 0x84 to 0xaf for nv50
+   // 0xc0 to 0xdf for nvc0
+   inline uint32_t getChipset() const { return chipset; }
+
+   virtual CodeEmitter *getCodeEmitter(Program::Type) = 0;
+
+   // Drivers should upload this so we can use it from all programs.
+   // The address chosen is supplied to the relocation routine.
+   virtual void getBuiltinCode(const uint32_t **code, uint32_t *size) const = 0;
+
+   virtual bool runLegalizePass(Program *, CGStage stage) const = 0;
+
+public:
+   struct OpInfo
+   {
+      OpInfo *variants;
+      operation op;
+      uint16_t srcTypes;
+      uint16_t dstTypes;
+      uint32_t immdBits;
+      uint8_t srcNr;
+      uint8_t srcMods[3];
+      uint8_t dstMods;
+      uint8_t srcFiles[3];
+      uint8_t dstFiles;
+      unsigned int minEncSize  : 4;
+      unsigned int vector      : 1;
+      unsigned int predicate   : 1;
+      unsigned int commutative : 1;
+      unsigned int pseudo      : 1;
+      unsigned int flow        : 1;
+      unsigned int hasDest     : 1;
+      unsigned int terminator  : 1;
+   };
+
+   inline const OpInfo& getOpInfo(const Instruction *) const;
+   inline const OpInfo& getOpInfo(const operation) const;
+
+   inline DataFile nativeFile(DataFile f) const;
+
+   virtual bool insnCanLoad(const Instruction *insn, int s,
+                            const Instruction *ld) const = 0;
+   virtual bool isOpSupported(operation, DataType) const = 0;
+   virtual bool isModSupported(const Instruction *,
+                               int s, Modifier) const = 0;
+   virtual bool isSatSupported(const Instruction *) const = 0;
+   virtual bool mayPredicate(const Instruction *,
+                             const Value *) const = 0;
+
+   virtual int getLatency(const Instruction *) const { return 1; }
+   virtual int getThroughput(const Instruction *) const { return 1; }
+
+   virtual unsigned int getFileSize(DataFile) const = 0;
+   virtual unsigned int getFileUnit(DataFile) const = 0;
+
+   virtual uint32_t getSVAddress(DataFile, const Symbol *) const = 0;
+
+public:
+   bool joinAnterior; // true if join is executed before the op
+
+   static const uint8_t operationSrcNr[OP_LAST + 1];
+
+protected:
+   uint32_t chipset;
+
+   DataFile nativeFileMap[DATA_FILE_COUNT];
+
+   OpInfo opInfo[OP_LAST + 1];
+};
+
+const Target::OpInfo& Target::getOpInfo(const Instruction *insn) const
+{
+   return opInfo[MIN2(insn->op, OP_LAST)];
+}
+
+const Target::OpInfo& Target::getOpInfo(const operation op) const
+{
+   return opInfo[op];
+}
+
+inline DataFile Target::nativeFile(DataFile f) const
+{
+   return nativeFileMap[f];
+}
+
+} // namespace nv50_ir
+
+#endif // __NV50_IR_TARGET_H__
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_util.cpp b/src/gallium/drivers/nv50/codegen/nv50_ir_util.cpp
new file mode 100644
index 00000000000..97f47a3ddbc
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_util.cpp
@@ -0,0 +1,253 @@
+
+#include "nv50_ir_util.h"
+
+namespace nv50_ir {
+
+void DLList::clear()
+{
+   for (Item *next, *item = head.next; item != &head; item = next) {
+      next = item->next;
+      delete item;
+   }
+   head.next = head.prev = &head;
+}
+
+void
+DLList::Iterator::erase()
+{
+   Item *rem = pos;
+
+   if (rem == term)
+      return;
+   pos = pos->next;
+
+   DLLIST_DEL(rem);
+   delete rem;
+}
+
+void DLList::Iterator::moveToList(DLList& dest)
+{
+   Item *item = pos;
+
+   assert(term != &dest.head);
+   assert(pos != term);
+
+   pos = pos->next;
+
+   DLLIST_DEL(item);
+   DLLIST_ADDHEAD(&dest.head, item);
+}
+
+bool
+DLList::Iterator::insert(void *data)
+{
+   Item *ins = new Item(data);
+
+   ins->next = pos->next;
+   ins->prev = pos;
+   pos->next->prev = ins;
+   pos->next = ins;
+
+   if (pos == term)
+      term = ins;
+
+   return true;
+}
+
+void
+Stack::moveTo(Stack& that)
+{
+   unsigned int newSize = this->size + that.size;
+
+   while (newSize > that.limit)
+      that.resize();
+   memcpy(&that.array[that.size], &array[0], this->size * sizeof(Item));
+
+   that.size = newSize;
+   this->size = 0;
+}
+
+Interval::~Interval()
+{
+   clear();
+}
+
+void
+Interval::clear()
+{
+   for (Range *next, *r = head; r; r = next) {
+      next = r->next;
+      delete r;
+   }
+}
+
+bool
+Interval::extend(int a, int b)
+{
+   Range *r, **nextp = &head;
+
+   // NOTE: we need empty intervals for fixed registers
+   // if (a == b)
+   //   return false;
+   assert(a <= b);
+
+   for (r = head; r; r = r->next) {
+      if (b < r->bgn)
+         break; // insert before
+      if (a > r->end) {
+         // insert after
+         nextp = &r->next;
+         continue;
+      }
+
+      // overlap
+      if (a < r->bgn) {
+         r->bgn = a;
+         if (b > r->end)
+            r->end = b;
+         r->coalesce(&tail);
+         return true;
+      }
+      if (b > r->end) {
+         r->end = b;
+         r->coalesce(&tail);
+         return true;
+      }
+      assert(a >= r->bgn);
+      assert(b <= r->end);
+      return true;
+   }
+
+   (*nextp) = new Range(a, b);
+   (*nextp)->next = r;
+
+   for (r = (*nextp); r->next; r = r->next);
+   tail = r;
+   return true;
+}
+
+bool Interval::contains(int pos)
+{
+   for (Range *r = head; r && r->bgn <= pos; r = r->next)
+      if (r->end > pos)
+         return true;
+   return false;
+}
+
+bool Interval::overlaps(const Interval &iv) const
+{
+   for (Range *rA = this->head; rA; rA = rA->next)
+      for (Range *rB = iv.head; rB; rB = rB->next)
+         if (rB->bgn < rA->end &&
+             rB->end > rA->bgn)
+            return true;
+   return false;
+}
+
+void Interval::unify(Interval &that)
+{
+   assert(this != &that);
+   for (Range *next, *r = that.head; r; r = next) {
+      next = r->next;
+      this->extend(r->bgn, r->end);
+      delete r;
+   }
+   that.head = NULL;
+}
+
+void Interval::print() const
+{
+   if (!head)
+      return;
+   INFO("[%i %i)", head->bgn, head->end);
+   for (const Range *r = head->next; r; r = r->next)
+      INFO(" [%i %i)", r->bgn, r->end);
+   INFO("\n");
+}
+
+void
+BitSet::andNot(const BitSet &set)
+{
+   assert(data && set.data);
+   assert(size >= set.size);
+   for (unsigned int i = 0; i < (set.size + 31) / 32; ++i)
+      data[i] &= ~set.data[i];
+}
+
+BitSet& BitSet::operator|=(const BitSet &set)
+{
+   assert(data && set.data);
+   assert(size >= set.size);
+   for (unsigned int i = 0; i < (set.size + 31) / 32; ++i)
+      data[i] |= set.data[i];
+   return *this;
+}
+
+bool BitSet::allocate(unsigned int nBits, bool zero)
+{
+   if (data && size < nBits) {
+      FREE(data);
+      data = NULL;
+   }
+   size = nBits;
+
+   if (!data)
+      data = reinterpret_cast<uint32_t *>(CALLOC((size + 31) / 32, 4));
+
+   if (zero)
+      memset(data, 0, (size + 7) / 8);
+   else
+      data[(size + 31) / 32 - 1] = 0; // clear unused bits (e.g. for popCount)
+
+   return data;
+}
+
+unsigned int BitSet::popCount() const
+{
+   unsigned int count = 0;
+
+   for (unsigned int i = 0; i < (size + 31) / 32; ++i)
+      if (data[i])
+         count += util_bitcount(data[i]);
+   return count;
+}
+
+void BitSet::fill(uint32_t val)
+{
+   unsigned int i;
+   for (i = 0; i < (size + 31) / 32; ++i)
+      data[i] = val;
+   if (val)
+      data[i] &= ~(0xffffffff << (size % 32)); // BE ?
+}
+
+void BitSet::setOr(BitSet *pA, BitSet *pB)
+{
+   if (!pB) {
+      *this = *pA;
+   } else {
+      for (unsigned int i = 0; i < (size + 31) / 32; ++i)
+         data[i] = pA->data[i] | pB->data[i];
+   }
+}
+
+void BitSet::print() const
+{
+   unsigned int n = 0;
+   INFO("BitSet of size %u:\n", size);
+   for (unsigned int i = 0; i < (size + 31) / 32; ++i) {
+      uint32_t bits = data[i];
+      while (bits) {
+         int pos = ffs(bits) - 1;
+         bits &= ~(1 << pos);
+         INFO(" %i", i * 32 + pos);
+         ++n;
+         if ((n % 16) == 0)
+            INFO("\n");
+      }
+   }
+   if (n % 16)
+      INFO("\n");
+}
+
+} // namespace nv50_ir
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_util.h b/src/gallium/drivers/nv50/codegen/nv50_ir_util.h
new file mode 100644
index 00000000000..2ffdcd65568
--- /dev/null
+++ b/src/gallium/drivers/nv50/codegen/nv50_ir_util.h
@@ -0,0 +1,585 @@
+
+#ifndef __NV50_IR_UTIL_H__
+#define __NV50_IR_UTIL_H__
+
+#include <new>
+#include <assert.h>
+#include <stdio.h>
+
+#include "util/u_inlines.h"
+#include "util/u_memory.h"
+
+#define ERROR(args...) debug_printf("ERROR: " args)
+#define WARN(args...) debug_printf("WARNING: " args)
+#define INFO(args...) debug_printf(args)
+
+#define INFO_DBG(m, f, args...)          \
+   do {                                  \
+      if (m & NV50_IR_DEBUG_##f)         \
+         debug_printf(args);             \
+   } while(0)
+
+#define FATAL(args...)          \
+   do {                         \
+      fprintf(stderr, args);    \
+      abort();                  \
+   } while(0)
+
+
+#define NV50_IR_FUNC_ALLOC_OBJ_DEF(obj, f, args...)               \
+   new ((f)->getProgram()->mem_##obj.allocate()) obj(f, args)
+
+#define new_Instruction(f, args...)                      \
+   NV50_IR_FUNC_ALLOC_OBJ_DEF(Instruction, f, args)
+#define new_CmpInstruction(f, args...)                   \
+   NV50_IR_FUNC_ALLOC_OBJ_DEF(CmpInstruction, f, args)
+#define new_TexInstruction(f, args...)                   \
+   NV50_IR_FUNC_ALLOC_OBJ_DEF(TexInstruction, f, args)
+#define new_FlowInstruction(f, args...)                  \
+   NV50_IR_FUNC_ALLOC_OBJ_DEF(FlowInstruction, f, args)
+
+#define new_LValue(f, args...)                  \
+   NV50_IR_FUNC_ALLOC_OBJ_DEF(LValue, f, args)
+
+
+#define NV50_IR_PROG_ALLOC_OBJ_DEF(obj, p, args...)   \
+   new ((p)->mem_##obj.allocate()) obj(p, args)
+
+#define new_Symbol(p, args...)                           \
+   NV50_IR_PROG_ALLOC_OBJ_DEF(Symbol, p, args)
+#define new_ImmediateValue(p, args...)                   \
+   NV50_IR_PROG_ALLOC_OBJ_DEF(ImmediateValue, p, args)
+
+
+#define delete_Instruction(p, insn) (p)->releaseInstruction(insn)
+#define delete_Value(p, val) (p)->releaseValue(val)
+
+
+namespace nv50_ir {
+
+class Iterator
+{
+public:
+   virtual void next() = 0;
+   virtual void *get() const = 0;
+   virtual bool end() const = 0; // if true, get will return 0
+};
+
+class ManipIterator : public Iterator
+{
+public:
+   virtual bool insert(void *) = 0; // insert after current position
+   virtual void erase() = 0;
+};
+
+// WARNING: do not use a->prev/next for __item or __list
+
+#define DLLIST_DEL(__item)                      \
+   do {                                         \
+      (__item)->prev->next = (__item)->next;    \
+      (__item)->next->prev = (__item)->prev;    \
+      (__item)->next = (__item);                \
+      (__item)->prev = (__item);                \
+   } while(0)
+
+#define DLLIST_ADDTAIL(__list, __item)          \
+   do {                                         \
+      (__item)->next = (__list);                \
+      (__item)->prev = (__list)->prev;          \
+      (__list)->prev->next = (__item);          \
+      (__list)->prev = (__item);                \
+   } while(0)
+
+#define DLLIST_ADDHEAD(__list, __item)          \
+   do {                                         \
+      (__item)->prev = (__list);                \
+      (__item)->next = (__list)->next;          \
+      (__list)->next->prev = (__item);          \
+      (__list)->next = (__item);                \
+   } while(0)
+
+#define DLLIST_MERGE(__listA, __listB, ty)      \
+   do {                                         \
+      ty prevB = (__listB)->prev;               \
+      (__listA)->prev->next = (__listB);        \
+      (__listB)->prev->next = (__listA);        \
+      (__listB)->prev = (__listA)->prev;        \
+      (__listA)->prev = prevB;                  \
+   } while(0)
+
+#define DLLIST_FOR_EACH(list, it) \
+   for (DLList::Iterator (it) = (list)->iterator(); !(it).end(); (it).next())
+
+class DLList
+{
+public:
+   class Item
+   {
+   public:
+      Item(void *priv) : next(this), prev(this), data(priv) { }
+
+   public:
+      Item *next;
+      Item *prev;
+      void *data;
+   };
+
+   DLList() : head(0) { }
+   ~DLList() { clear(); }
+
+   inline void insertHead(void *data)
+   {
+      Item *item = new Item(data);
+
+      assert(data);
+
+      item->prev = &head;
+      item->next = head.next;
+      head.next->prev = item;
+      head.next = item;
+   }
+
+   inline void insertTail(void *data)
+   {
+      Item *item = new Item(data);
+
+      assert(data);
+
+      DLLIST_ADDTAIL(&head, item);
+   }
+
+   inline void insert(void *data) { insertTail(data); }
+
+   void clear();
+
+   class Iterator : public ManipIterator
+   {
+   public:
+      Iterator(Item *head, bool r) : rev(r), pos(r ? head->prev : head->next),
+                                     term(head) { }
+
+      virtual void next() { if (!end()) pos = rev ? pos->prev : pos->next; }
+      virtual void *get() const { return pos->data; }
+      virtual bool end() const { return pos == term; }
+
+      // caution: if you're at end-2 and erase it, then do next, you're at end
+      virtual void erase();
+      virtual bool insert(void *data);
+
+      // move item to a another list, no consistency with its iterators though
+      void moveToList(DLList&);
+
+   private:
+      const bool rev;
+      Item *pos;
+      Item *term;
+
+      friend class DLList;
+   };
+
+   inline void erase(Iterator& pos)
+   {
+      pos.erase();
+   }
+
+   Iterator iterator()
+   {
+      return Iterator(&head, false);
+   }
+
+   Iterator revIterator()
+   {
+      return Iterator(&head, true);
+   }
+
+private:
+   Item head;
+};
+
+class Stack
+{
+public:
+   class Item {
+   public:
+      union {
+         void *p;
+         int i;
+         unsigned int u;
+         float f;
+         double d;
+      } u;
+
+      Item() { memset(&u, 0, sizeof(u)); }
+   };
+   
+   Stack() : size(0), limit(0), array(0) { }
+   ~Stack() { if (array) FREE(array); }
+
+   inline void push(int i)          { Item data; data.u.i = i; push(data); }
+   inline void push(unsigned int u) { Item data; data.u.u = u; push(data); }
+   inline void push(void *p)        { Item data; data.u.p = p; push(data); }
+   inline void push(float f)        { Item data; data.u.f = f; push(data); }
+
+   inline void push(Item data)
+   {
+      if (size == limit)
+         resize();
+      array[size++] = data;
+   }
+
+   inline Item pop()
+   {
+      if (!size) {
+         Item data;
+         assert(0);
+         return data;
+      }
+      return array[--size];
+   }
+
+   inline unsigned int getSize() { return size; }
+
+   inline Item& peek() { assert(size); return array[size - 1]; }
+
+   void clear(bool releaseStorage = false)
+   {
+      if (releaseStorage && array)
+         FREE(array);
+      size = limit = 0;
+   }
+
+   void moveTo(Stack&); // move all items to target (not like push(pop()))
+
+private:
+   void resize()
+   {
+         unsigned int sizeOld, sizeNew;
+
+         sizeOld = limit * sizeof(Item);
+         limit = MAX2(4, limit + limit);
+         sizeNew = limit * sizeof(Item);
+
+         array = (Item *)REALLOC(array, sizeOld, sizeNew);
+   }
+
+   unsigned int size;
+   unsigned int limit;
+   Item *array;
+};
+
+class DynArray
+{
+public:
+   class Item
+   {
+   public:
+      union {
+         uint32_t u32;
+         void *p;
+      };
+   };
+
+   DynArray() : data(NULL), size(0) { }
+
+   ~DynArray() { if (data) FREE(data); }
+
+   inline Item& operator[](unsigned int i)
+   {
+      if (i >= size)
+         resize(i);
+      return data[i];
+   }
+
+   inline const Item operator[](unsigned int i) const
+   {
+      return data[i];
+   }
+
+   void resize(unsigned int index)
+   {
+      const unsigned int oldSize = size * sizeof(Item);
+
+      if (!size)
+         size = 8;
+      while (size <= index)
+         size <<= 1;
+
+      data = (Item *)REALLOC(data, oldSize, size * sizeof(Item));
+   }
+
+private:
+   Item *data;
+   unsigned int size;
+};
+
+class ArrayList
+{
+public:
+   ArrayList() : size(0) { }
+
+   void insert(void *item, int& id)
+   {
+      id = ids.getSize() ? ids.pop().u.i : size++;
+      data[id].p = item;
+   }
+
+   void remove(int& id)
+   {
+      const unsigned int uid = id;
+      assert(uid < size && data[id].p);
+      ids.push(uid);
+      data[uid].p = NULL;
+      id = -1;
+   }
+
+   inline int getSize() const { return size; }
+
+   inline void *get(unsigned int id) { assert(id < size); return data[id].p; }
+
+   class Iterator : public nv50_ir::Iterator
+   {
+   public:
+      Iterator(const ArrayList *array) : pos(0), data(array->data)
+      {
+         size = array->getSize();
+         if (size)
+            nextValid();
+      }
+
+      void nextValid() { while ((pos < size) && !data[pos].p) ++pos; }
+
+      void next() { if (pos < size) { ++pos; nextValid(); } }
+      void *get() const { assert(pos < size); return data[pos].p; }
+      bool end() const { return pos >= size; }
+
+   private:
+      unsigned int pos;
+      unsigned int size;
+      const DynArray& data;
+
+      friend class ArrayList;
+   };
+
+   Iterator iterator() const { return Iterator(this); }
+
+private:
+   DynArray data;
+   Stack ids;
+   unsigned int size;
+};
+
+class Interval
+{
+public:
+   Interval() : head(0), tail(0) { }
+   ~Interval();
+
+   bool extend(int, int);
+   void unify(Interval&); // clears source interval
+   void clear();
+
+   inline int begin() { return head ? head->bgn : -1; }
+   inline int end() { checkTail(); return tail ? tail->end : -1; }
+   inline bool isEmpty() const { return !head; }
+   bool overlaps(const Interval&) const;
+   bool contains(int pos);
+
+   void print() const;
+
+   inline void checkTail() const;
+
+private:
+   class Range
+   {
+   public:
+      Range(int a, int b) : next(0), bgn(a), end(b) { }
+
+      Range *next;
+      int bgn;
+      int end;
+
+      void coalesce(Range **ptail)
+      {
+         Range *rnn;
+
+         while (next && end >= next->bgn) {
+            assert(bgn <= next->bgn);
+            rnn = next->next;
+            end = MAX2(end, next->end);
+            delete next;
+            next = rnn;
+         }
+         if (!next)
+            *ptail = this;
+      }
+   };
+
+   Range *head;
+   Range *tail;
+};
+
+class BitSet
+{
+public:
+   BitSet() : marker(false), data(0), size(0) { }
+   BitSet(unsigned int nBits, bool zero) : marker(false), data(0), size(0)
+   {
+      allocate(nBits, zero);
+   }
+   ~BitSet()
+   {
+      if (data)
+         FREE(data);
+   }
+
+   bool allocate(unsigned int nBits, bool zero);
+
+   inline unsigned int getSize() const { return size; }
+
+   void fill(uint32_t val);
+
+   void setOr(BitSet *, BitSet *); // second BitSet may be NULL
+
+   inline void set(unsigned int i)
+   {
+      assert(i < size);
+      data[i / 32] |= 1 << (i % 32);
+   }
+
+   inline void clr(unsigned int i)
+   {
+      assert(i < size);
+      data[i / 32] &= ~(1 << (i % 32));
+   }
+
+   inline bool test(unsigned int i) const
+   {
+      assert(i < size);
+      return data[i / 32] & (1 << (i % 32));
+   }
+
+   BitSet& operator|=(const BitSet&);
+
+   BitSet& operator=(const BitSet& set)
+   {
+      assert(data && set.data);
+      assert(size == set.size);
+      memcpy(data, set.data, (set.size + 7) / 8);
+      return *this;
+   }
+
+   void andNot(const BitSet&);
+
+   unsigned int popCount() const;
+
+   void print() const;
+
+public:
+   bool marker; // for user
+
+private:
+   uint32_t *data;
+   unsigned int size;
+};
+
+void Interval::checkTail() const
+{
+#if NV50_DEBUG & NV50_DEBUG_PROG_RA
+   Range *r = head;
+   while (r->next)
+      r = r->next;
+   assert(tail == r);
+#endif
+}
+
+class MemoryPool
+{
+private:
+   inline bool enlargeAllocationsArray(const unsigned int id, unsigned int nr)
+   {
+      const unsigned int size = sizeof(uint8_t *) * id;
+      const unsigned int incr = sizeof(uint8_t *) * nr;
+
+      uint8_t **alloc = (uint8_t **)REALLOC(allocArray, size, size + incr);
+      if (!alloc)
+         return false;
+      allocArray = alloc;
+      return true;
+   }
+
+   inline bool enlargeCapacity()
+   {
+      const unsigned int id = count >> objStepLog2;
+
+      uint8_t *const mem = (uint8_t *)MALLOC(objSize << objStepLog2);
+      if (!mem)
+         return false;
+
+      if (!(id % 32)) {
+         if (!enlargeAllocationsArray(id, 32)) {
+            FREE(mem);
+            return false;
+         }
+      }
+      allocArray[id] = mem;
+      return true;
+   }
+
+public:
+   MemoryPool(unsigned int size, unsigned int incr) : objSize(size),
+                                                      objStepLog2(incr)
+   {
+      allocArray = NULL;
+      released = NULL;
+      count = 0;
+   }
+
+   ~MemoryPool()
+   {
+      unsigned int allocCount = (count + (1 << objStepLog2) - 1) >> objStepLog2;
+      for (unsigned int i = 0; i < allocCount && allocArray[i]; ++i)
+         FREE(allocArray[i]);
+      if (allocArray)
+         FREE(allocArray);
+   }
+
+   void *allocate()
+   {
+      void *ret;
+      const unsigned int mask = (1 << objStepLog2) - 1;
+
+      if (released) {
+         ret = released;
+         released = *(void **)released;
+         return ret;
+      }
+
+      if (!(count & mask))
+         if (!enlargeCapacity())
+            return NULL;
+
+      ret = allocArray[count >> objStepLog2] + (count & mask) * objSize;
+      ++count;
+      return ret;
+   }
+
+   void release(void *ptr)
+   {
+      *(void **)ptr = released;
+      released = ptr;
+   }
+
+private:
+   uint8_t **allocArray; // array (list) of MALLOC allocations
+
+   void *released; // list of released objects
+
+   unsigned int count; // highest allocated object
+
+   const unsigned int objSize;
+   const unsigned int objStepLog2;
+};
+
+} // namespace nv50_ir
+
+#endif // __NV50_IR_UTIL_H__