diff options
8 files changed, 738 insertions, 15 deletions
diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir.h b/src/gallium/drivers/nv50/codegen/nv50_ir.h index da9042066ad..e544d071b52 100644 --- a/src/gallium/drivers/nv50/codegen/nv50_ir.h +++ b/src/gallium/drivers/nv50/codegen/nv50_ir.h @@ -681,6 +681,8 @@ public: uint8_t subOp; // quadop, 1 for mul-high, etc. + uint8_t sched; // scheduling data (NOTE: maybe move to separate storage) + unsigned encSize : 4; // encoding size in bytes unsigned saturate : 1; // to [0.0f, 1.0f] unsigned join : 1; // converge control flow (use OP_JOIN until end) diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_print.cpp b/src/gallium/drivers/nv50/codegen/nv50_ir_print.cpp index 4652bb95f69..9d92b7bc24b 100644 --- a/src/gallium/drivers/nv50/codegen/nv50_ir_print.cpp +++ b/src/gallium/drivers/nv50/codegen/nv50_ir_print.cpp @@ -53,6 +53,26 @@ static const char *colour[8] = #endif }; +static const char *OpClassStr[OPCLASS_OTHER + 1] = +{ + "MOVE", + "LOAD", + "STORE", + "ARITH", + "SHIFT", + "SFU", + "LOGIC", + "COMPARE", + "CONVERT", + "ATOMIC", + "TEXTURE", + "SURFACE", + "FLOW", + "(INVALID)", + "PSEUDO", + "OTHER" +}; + const char *operationStr[OP_LAST + 1] = { "nop", diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_target.cpp b/src/gallium/drivers/nv50/codegen/nv50_ir_target.cpp index e3eae69554c..f718912fb39 100644 --- a/src/gallium/drivers/nv50/codegen/nv50_ir_target.cpp +++ b/src/gallium/drivers/nv50/codegen/nv50_ir_target.cpp @@ -52,6 +52,65 @@ const uint8_t Target::operationSrcNr[OP_LAST + 1] = 0 }; +const OpClass Target::operationClass[OP_LAST + 1] = +{ + // NOP; PHI; UNION, SPLIT, MERGE, CONSTRAINT + OPCLASS_OTHER, + OPCLASS_PSEUDO, + OPCLASS_PSEUDO, OPCLASS_PSEUDO, OPCLASS_PSEUDO, OPCLASS_PSEUDO, + // MOV; LOAD; STORE + OPCLASS_MOVE, + OPCLASS_LOAD, + OPCLASS_STORE, + // ADD, SUB, MUL; DIV, MOD; MAD, FMA, SAD + OPCLASS_ARITH, OPCLASS_ARITH, OPCLASS_ARITH, + OPCLASS_ARITH, OPCLASS_ARITH, + OPCLASS_ARITH, OPCLASS_ARITH, OPCLASS_ARITH, + // ABS, NEG; NOT, AND, OR, XOR; SHL, SHR + OPCLASS_CONVERT, OPCLASS_CONVERT, + OPCLASS_LOGIC, OPCLASS_LOGIC, OPCLASS_LOGIC, OPCLASS_LOGIC, + OPCLASS_SHIFT, OPCLASS_SHIFT, + // MAX, MIN + OPCLASS_COMPARE, OPCLASS_COMPARE, + // SAT, CEIL, FLOOR, TRUNC; CVT + OPCLASS_CONVERT, OPCLASS_CONVERT, OPCLASS_CONVERT, OPCLASS_CONVERT, + OPCLASS_CONVERT, + // SET(AND,OR,XOR); SELP, SLCT + OPCLASS_COMPARE, OPCLASS_COMPARE, OPCLASS_COMPARE, OPCLASS_COMPARE, + OPCLASS_COMPARE, OPCLASS_COMPARE, + // RCP, RSQ, LG2, SIN, COS; EX2, EXP, LOG, PRESIN, PREEX2; SQRT, POW + OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU, + OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU, + OPCLASS_SFU, OPCLASS_SFU, + // BRA, CALL, RET; CONT, BREAK, PRE(RET,CONT,BREAK); BRKPT, JOINAT, JOIN + OPCLASS_FLOW, OPCLASS_FLOW, OPCLASS_FLOW, + OPCLASS_FLOW, OPCLASS_FLOW, OPCLASS_FLOW, OPCLASS_FLOW, OPCLASS_FLOW, + OPCLASS_FLOW, OPCLASS_FLOW, OPCLASS_FLOW, + // DISCARD, EXIT + OPCLASS_FLOW, OPCLASS_FLOW, + // MEMBAR + OPCLASS_OTHER, + // VFETCH, PFETCH, EXPORT + OPCLASS_LOAD, OPCLASS_OTHER, OPCLASS_STORE, + // LINTERP, PINTERP + OPCLASS_SFU, OPCLASS_SFU, + // EMIT, RESTART + OPCLASS_OTHER, OPCLASS_OTHER, + // TEX, TXB, TXL, TXF; TXQ, TXD, TXG, TEXCSAA + OPCLASS_TEXTURE, OPCLASS_TEXTURE, OPCLASS_TEXTURE, OPCLASS_TEXTURE, + OPCLASS_TEXTURE, OPCLASS_TEXTURE, OPCLASS_TEXTURE, OPCLASS_TEXTURE, + // SULD, SUST + OPCLASS_SURFACE, OPCLASS_SURFACE, + // DFDX, DFDY, RDSV, WRSV; PIXLD, QUADOP, QUADON, QUADPOP + OPCLASS_OTHER, OPCLASS_OTHER, OPCLASS_OTHER, OPCLASS_OTHER, + OPCLASS_OTHER, OPCLASS_OTHER, OPCLASS_OTHER, OPCLASS_OTHER, + // POPCNT, INSBF, EXTBF + OPCLASS_OTHER, OPCLASS_OTHER, OPCLASS_OTHER, + // TEXBAR + OPCLASS_OTHER, + OPCLASS_PSEUDO // LAST +}; + extern Target *getTargetNVC0(unsigned int chipset); extern Target *getTargetNV50(unsigned int chipset); @@ -104,6 +163,11 @@ CodeEmitter::printBinary() const INFO("\n"); } +static inline uint32_t sizeToBundlesNVE4(uint32_t size) +{ + return (size + 55) / 56; +} + void CodeEmitter::prepareEmission(Program *prog) { @@ -112,6 +176,23 @@ CodeEmitter::prepareEmission(Program *prog) Function *func = reinterpret_cast<Function *>(fi.get()); func->binPos = prog->binSize; prepareEmission(func); + + // adjust sizes & positions for schedulding info: + if (prog->getTarget()->hasSWSched) { + BasicBlock *bb = NULL; + for (int i = 0; i < func->bbCount; ++i) { + bb = func->bbArray[i]; + const uint32_t oldPos = bb->binPos; + const uint32_t oldEnd = bb->binPos + bb->binSize; + uint32_t adjPos = oldPos + sizeToBundlesNVE4(oldPos) * 8; + uint32_t adjEnd = oldEnd + sizeToBundlesNVE4(oldEnd) * 8; + bb->binPos = adjPos; + bb->binSize = adjEnd - adjPos; + } + if (bb) + func->binSize = bb->binPos + bb->binSize; + } + prog->binSize += func->binSize; } } diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_target.h b/src/gallium/drivers/nv50/codegen/nv50_ir_target.h index 88996ebbde3..c60ee0216f7 100644 --- a/src/gallium/drivers/nv50/codegen/nv50_ir_target.h +++ b/src/gallium/drivers/nv50/codegen/nv50_ir_target.h @@ -93,9 +93,31 @@ protected: RelocInfo *relocInfo; }; + +enum OpClass +{ + OPCLASS_MOVE = 0, + OPCLASS_LOAD = 1, + OPCLASS_STORE = 2, + OPCLASS_ARITH = 3, + OPCLASS_SHIFT = 4, + OPCLASS_SFU = 5, + OPCLASS_LOGIC = 6, + OPCLASS_COMPARE = 7, + OPCLASS_CONVERT = 8, + OPCLASS_ATOMIC = 9, + OPCLASS_TEXTURE = 10, + OPCLASS_SURFACE = 11, + OPCLASS_FLOW = 12, + OPCLASS_PSEUDO = 14, + OPCLASS_OTHER = 15 +}; + class Target { public: + Target(bool j, bool s) : joinAnterior(j), hasSWSched(s) { } + static Target *create(uint32_t chipset); static void destroy(Target *); @@ -153,6 +175,9 @@ public: virtual bool mayPredicate(const Instruction *, const Value *) const = 0; + // whether @insn can be issued together with @next (order matters) + virtual bool canDualIssue(const Instruction *insn, + const Instruction *next) const { return false; } virtual int getLatency(const Instruction *) const { return 1; } virtual int getThroughput(const Instruction *) const { return 1; } @@ -162,9 +187,20 @@ public: virtual uint32_t getSVAddress(DataFile, const Symbol *) const = 0; public: - bool joinAnterior; // true if join is executed before the op + const bool joinAnterior; // true if join is executed before the op + const bool hasSWSched; // true if code should provide scheduling data static const uint8_t operationSrcNr[OP_LAST + 1]; + static const OpClass operationClass[OP_LAST + 1]; + + static inline uint8_t getOpSrcNr(operation op) + { + return operationSrcNr[op]; + } + static inline OpClass getOpClass(operation op) + { + return operationClass[op]; + } protected: uint32_t chipset; diff --git a/src/gallium/drivers/nv50/codegen/nv50_ir_target_nv50.cpp b/src/gallium/drivers/nv50/codegen/nv50_ir_target_nv50.cpp index a64f7f72255..5e541e514cb 100644 --- a/src/gallium/drivers/nv50/codegen/nv50_ir_target_nv50.cpp +++ b/src/gallium/drivers/nv50/codegen/nv50_ir_target_nv50.cpp @@ -29,7 +29,7 @@ Target *getTargetNV50(unsigned int chipset) return new TargetNV50(chipset); } -TargetNV50::TargetNV50(unsigned int card) +TargetNV50::TargetNV50(unsigned int card) : Target(true, false) { chipset = card; @@ -132,8 +132,6 @@ void TargetNV50::initOpInfo() OP_CALL, OP_PREBREAK, OP_PRERET, OP_QUADON, OP_QUADPOP, OP_JOINAT }; - joinAnterior = true; - for (i = 0; i < DATA_FILE_COUNT; ++i) nativeFileMap[i] = (DataFile)i; nativeFileMap[FILE_PREDICATE] = FILE_FLAGS; diff --git a/src/gallium/drivers/nvc0/codegen/nv50_ir_emit_nvc0.cpp b/src/gallium/drivers/nvc0/codegen/nv50_ir_emit_nvc0.cpp index 48d99bfe803..c91c56c9c10 100644 --- a/src/gallium/drivers/nvc0/codegen/nv50_ir_emit_nvc0.cpp +++ b/src/gallium/drivers/nvc0/codegen/nv50_ir_emit_nvc0.cpp @@ -33,6 +33,7 @@ public: virtual bool emitInstruction(Instruction *); virtual uint32_t getMinEncodingSize(const Instruction *) const; + virtual void prepareEmission(Function *); inline void setProgramType(Program::Type pType) { progType = pType; } @@ -41,6 +42,8 @@ private: Program::Type progType; + const bool writeIssueDelays; + private: void emitForm_A(const Instruction *, uint64_t); void emitForm_B(const Instruction *, uint64_t); @@ -1505,15 +1508,40 @@ CodeEmitterNVC0::emitMOV(const Instruction *i) bool CodeEmitterNVC0::emitInstruction(Instruction *insn) { + unsigned int size = insn->encSize; + + if (writeIssueDelays && !(codeSize & 0x3f)) + size += 8; + if (!insn->encSize) { ERROR("skipping unencodable instruction: "); insn->print(); return false; } else - if (codeSize + insn->encSize > codeSizeLimit) { + if (codeSize + size > codeSizeLimit) { ERROR("code emitter output buffer too small\n"); return false; } + if (writeIssueDelays) { + if (!(codeSize & 0x3f)) { + code[0] = 0x00000007; // cf issue delay "instruction" + code[1] = 0x20000000; + code += 2; + codeSize += 8; + } + const unsigned int id = (codeSize & 0x3f) / 8 - 1; + uint32_t *data = code - (id * 2 + 2); + if (id <= 2) { + data[0] |= insn->sched << (id * 8 + 4); + } else + if (id == 3) { + data[0] |= insn->sched << 28; + data[1] |= insn->sched >> 4; + } else { + data[1] |= insn->sched << ((id - 4) * 8 + 4); + } + } + // assert that instructions with multiple defs don't corrupt registers for (int d = 0; insn->defExists(d); ++d) assert(insn->asTex() || insn->def(d).rep()->reg.data.id >= 0); @@ -1707,7 +1735,7 @@ CodeEmitterNVC0::getMinEncodingSize(const Instruction *i) const { const Target::OpInfo &info = targ->getOpInfo(i); - if (info.minEncSize == 8 || 1) + if (writeIssueDelays || info.minEncSize == 8 || 1) return 8; if (i->ftz || i->saturate || i->join) @@ -1761,7 +1789,503 @@ CodeEmitterNVC0::getMinEncodingSize(const Instruction *i) const return 4; } -CodeEmitterNVC0::CodeEmitterNVC0(const TargetNVC0 *target) : CodeEmitter(target) +// Simplified, erring on safe side. +class SchedDataCalculator : public Pass +{ +public: + SchedDataCalculator(const Target *targ) : targ(targ) { } + +private: + struct RegScores + { + struct Resource { + int st[DATA_FILE_COUNT]; // LD to LD delay 3 + int ld[DATA_FILE_COUNT]; // ST to ST delay 3 + int tex; // TEX to non-TEX delay 17 (0x11) + int sfu; // SFU to SFU delay 3 (except PRE-ops) + int imul; // integer MUL to MUL delay 3 + } res; + struct ScoreData { + int r[64]; + int p[8]; + int c; + } rd, wr; + int base; + + void rebase(const int base) + { + const int delta = this->base - base; + if (!delta) + return; + this->base = 0; + + for (int i = 0; i < 64; ++i) { + rd.r[i] += delta; + wr.r[i] += delta; + } + for (int i = 0; i < 8; ++i) { + rd.p[i] += delta; + wr.p[i] += delta; + } + rd.c += delta; + wr.c += delta; + + for (unsigned int f = 0; f < DATA_FILE_COUNT; ++f) { + res.ld[f] += delta; + res.st[f] += delta; + } + res.sfu += delta; + res.imul += delta; + res.tex += delta; + } + void wipe() + { + memset(&rd, 0, sizeof(rd)); + memset(&wr, 0, sizeof(wr)); + memset(&res, 0, sizeof(res)); + } + int getLatest(const ScoreData& d) const + { + int max = 0; + for (int i = 0; i < 64; ++i) + if (d.r[i] > max) + max = d.r[i]; + for (int i = 0; i < 8; ++i) + if (d.p[i] > max) + max = d.p[i]; + if (d.c > max) + max = d.c; + return max; + } + inline int getLatestRd() const + { + return getLatest(rd); + } + inline int getLatestWr() const + { + return getLatest(wr); + } + inline int getLatest() const + { + const int a = getLatestRd(); + const int b = getLatestWr(); + + int max = MAX2(a, b); + for (unsigned int f = 0; f < DATA_FILE_COUNT; ++f) { + max = MAX2(res.ld[f], max); + max = MAX2(res.st[f], max); + } + max = MAX2(res.sfu, max); + max = MAX2(res.imul, max); + max = MAX2(res.tex, max); + return max; + } + void setMax(const RegScores *that) + { + for (int i = 0; i < 64; ++i) { + rd.r[i] = MAX2(rd.r[i], that->rd.r[i]); + wr.r[i] = MAX2(wr.r[i], that->wr.r[i]); + } + for (int i = 0; i < 8; ++i) { + rd.p[i] = MAX2(rd.p[i], that->rd.p[i]); + wr.p[i] = MAX2(wr.p[i], that->wr.p[i]); + } + rd.c = MAX2(rd.c, that->rd.c); + wr.c = MAX2(wr.c, that->wr.c); + + for (unsigned int f = 0; f < DATA_FILE_COUNT; ++f) { + res.ld[f] = MAX2(res.ld[f], that->res.ld[f]); + res.st[f] = MAX2(res.st[f], that->res.st[f]); + } + res.sfu = MAX2(res.sfu, that->res.sfu); + res.imul = MAX2(res.imul, that->res.imul); + res.tex = MAX2(res.tex, that->res.tex); + } + void print(int cycle) + { + for (int i = 0; i < 64; ++i) { + if (rd.r[i] > cycle) + INFO("rd $r%i @ %i\n", i, rd.r[i]); + if (wr.r[i] > cycle) + INFO("wr $r%i @ %i\n", i, wr.r[i]); + } + for (int i = 0; i < 8; ++i) { + if (rd.p[i] > cycle) + INFO("rd $p%i @ %i\n", i, rd.p[i]); + if (wr.p[i] > cycle) + INFO("wr $p%i @ %i\n", i, wr.p[i]); + } + if (rd.c > cycle) + INFO("rd $c @ %i\n", rd.c); + if (wr.c > cycle) + INFO("wr $c @ %i\n", wr.c); + if (res.sfu > cycle) + INFO("sfu @ %i\n", res.sfu); + if (res.imul > cycle) + INFO("imul @ %i\n", res.imul); + if (res.tex > cycle) + INFO("tex @ %i\n", res.tex); + } + }; + + RegScores *score; // for current BB + std::vector<RegScores> scoreBoards; + int cycle; + int prevData; + operation prevOp; + + const Target *targ; + + bool visit(Function *); + bool visit(BasicBlock *); + + void commitInsn(const Instruction *, int cycle); + int calcDelay(const Instruction *, int cycle) const; + void setDelay(Instruction *, int delay, Instruction *next); + + void recordRd(const Value *, const int ready); + void recordWr(const Value *, const int ready); + void checkRd(const Value *, int cycle, int& delay) const; + void checkWr(const Value *, int cycle, int& delay) const; + + int getCycles(const Instruction *, int origDelay) const; +}; + +void +SchedDataCalculator::setDelay(Instruction *insn, int delay, Instruction *next) +{ + if (insn->op == OP_EXIT) + delay = MAX2(delay, 14); + + if (insn->op == OP_TEXBAR) { + // TODO: except if results not used before EXIT + insn->sched = 0xc2; + } else + if (insn->op == OP_JOIN || insn->join) { + insn->sched = 0x00; + } else + if (delay >= 0 || prevData == 0x04 || + !next || !targ->canDualIssue(insn, next)) { + insn->sched = static_cast<uint8_t>(MAX2(delay, 0)); + if (prevOp == OP_EXPORT) + insn->sched |= 0x40; + else + insn->sched |= 0x20; + } else { + insn->sched = 0x04; // dual-issue + } + + if (prevData != 0x04 || prevOp != OP_EXPORT) + if (insn->sched != 0x04 || insn->op == OP_EXPORT) + prevOp = insn->op; + + prevData = insn->sched; +} + +int +SchedDataCalculator::getCycles(const Instruction *insn, int origDelay) const +{ + if (insn->sched & 0x80) { + int c = (insn->sched & 0x0f) * 2 + 1; + if (insn->op == OP_TEXBAR && origDelay > 0) + c += origDelay; + return c; + } + if (insn->sched & 0x60) + return (insn->sched & 0x1f) + 1; + return (insn->sched == 0x04) ? 0 : 32; +} + +bool +SchedDataCalculator::visit(Function *func) +{ + scoreBoards.resize(func->cfg.getSize()); + for (size_t i = 0; i < scoreBoards.size(); ++i) + scoreBoards[i].wipe(); + return true; +} + +bool +SchedDataCalculator::visit(BasicBlock *bb) +{ + Instruction *insn; + Instruction *next = NULL; + + int cycle = 0; + + prevData = 0x00; + prevOp = OP_NOP; + score = &scoreBoards.at(bb->getId()); + + for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) { + BasicBlock *in = BasicBlock::get(ei.getNode()); + if (in->getExit()) { + if (prevData != 0x04) + prevData = in->getExit()->sched; + prevOp = in->getExit()->op; + } + if (ei.getType() != Graph::Edge::BACK) + score->setMax(&scoreBoards.at(in->getId())); + // back branches will wait until all target dependencies are satisfied + } + if (bb->cfg.incidentCount() > 1) + prevOp = OP_NOP; + +#ifdef NVC0_DEBUG_SCHED_DATA + INFO("=== BB:%i initial scores\n", bb->getId()); + score->print(cycle); +#endif + + for (insn = bb->getEntry(); insn && insn->next; insn = insn->next) { + next = insn->next; + + commitInsn(insn, cycle); + int delay = calcDelay(next, cycle); + setDelay(insn, delay, next); + cycle += getCycles(insn, delay); + +#ifdef NVC0_DEBUG_SCHED_DATA + INFO("cycle %i, sched %02x\n", cycle, insn->sched); + insn->print(); + next->print(); +#endif + } + if (!insn) + return true; + commitInsn(insn, cycle); + + int bbDelay = -1; + + for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) { + BasicBlock *out = BasicBlock::get(ei.getNode()); + + if (ei.getType() != Graph::Edge::BACK) { + // only test the first instruction of the outgoing block + next = out->getEntry(); + if (next) + bbDelay = MAX2(bbDelay, calcDelay(next, cycle)); + } else { + // wait until all dependencies are satisfied + const int regsFree = score->getLatest(); + next = out->getFirst(); + for (int c = cycle; next && c < regsFree; next = next->next) { + bbDelay = MAX2(bbDelay, calcDelay(next, c)); + c += getCycles(next, bbDelay); + } + next = NULL; + } + } + if (bb->cfg.outgoingCount() != 1) + next = NULL; + setDelay(insn, bbDelay, next); + cycle += getCycles(insn, bbDelay); + + score->rebase(cycle); // common base for initializing out blocks' scores + return true; +} + +#define NVE4_MAX_ISSUE_DELAY 0x1f +int +SchedDataCalculator::calcDelay(const Instruction *insn, int cycle) const +{ + int delay = 0, ready = cycle; + + for (int s = 0; insn->srcExists(s); ++s) + checkRd(insn->getSrc(s), cycle, delay); + // WAR & WAW don't seem to matter + // for (int s = 0; insn->srcExists(s); ++s) + // recordRd(insn->getSrc(s), cycle); + + switch (Target::getOpClass(insn->op)) { + case OPCLASS_SFU: + ready = score->res.sfu; + break; + case OPCLASS_ARITH: + if (insn->op == OP_MUL && !isFloatType(insn->dType)) + ready = score->res.imul; + break; + case OPCLASS_TEXTURE: + ready = score->res.tex; + break; + case OPCLASS_LOAD: + ready = score->res.ld[insn->src(0).getFile()]; + break; + case OPCLASS_STORE: + ready = score->res.st[insn->src(0).getFile()]; + break; + default: + break; + } + if (Target::getOpClass(insn->op) != OPCLASS_TEXTURE) + ready = MAX2(ready, score->res.tex); + + delay = MAX2(delay, ready - cycle); + + // if can issue next cycle, delay is 0, not 1 + return MIN2(delay - 1, NVE4_MAX_ISSUE_DELAY); +} + +void +SchedDataCalculator::commitInsn(const Instruction *insn, int cycle) +{ + const int ready = cycle + targ->getLatency(insn); + + for (int d = 0; insn->defExists(d); ++d) + recordWr(insn->getDef(d), ready); + // WAR & WAW don't seem to matter + // for (int s = 0; insn->srcExists(s); ++s) + // recordRd(insn->getSrc(s), cycle); + + switch (Target::getOpClass(insn->op)) { + case OPCLASS_SFU: + score->res.sfu = cycle + 4; + break; + case OPCLASS_ARITH: + if (insn->op == OP_MUL && !isFloatType(insn->dType)) + score->res.imul = cycle + 4; + break; + case OPCLASS_TEXTURE: + score->res.tex = cycle + 18; + break; + case OPCLASS_LOAD: + if (insn->src(0).getFile() == FILE_MEMORY_CONST) + break; + score->res.ld[insn->src(0).getFile()] = cycle + 4; + score->res.st[insn->src(0).getFile()] = ready; + break; + case OPCLASS_STORE: + score->res.st[insn->src(0).getFile()] = cycle + 4; + score->res.ld[insn->src(0).getFile()] = ready; + break; + case OPCLASS_OTHER: + if (insn->op == OP_TEXBAR) + score->res.tex = cycle; + break; + default: + break; + } + +#ifdef NVC0_DEBUG_SCHED_DATA + score->print(cycle); +#endif +} + +void +SchedDataCalculator::checkRd(const Value *v, int cycle, int& delay) const +{ + int ready = cycle; + int a, b; + + switch (v->reg.file) { + case FILE_GPR: + a = v->reg.data.id; + b = a + v->reg.size / 4; + for (int r = a; r < b; ++r) + ready = MAX2(ready, score->rd.r[r]); + break; + case FILE_PREDICATE: + ready = MAX2(ready, score->rd.p[v->reg.data.id]); + break; + case FILE_FLAGS: + ready = MAX2(ready, score->rd.c); + break; + case FILE_SHADER_INPUT: + case FILE_SHADER_OUTPUT: // yes, TCPs can read outputs + case FILE_MEMORY_LOCAL: + case FILE_MEMORY_CONST: + case FILE_MEMORY_SHARED: + case FILE_MEMORY_GLOBAL: + case FILE_SYSTEM_VALUE: + // TODO: any restrictions here ? + break; + case FILE_IMMEDIATE: + break; + default: + assert(0); + break; + } + if (cycle < ready) + delay = MAX2(delay, ready - cycle); +} + +void +SchedDataCalculator::checkWr(const Value *v, int cycle, int& delay) const +{ + int ready = cycle; + int a, b; + + switch (v->reg.file) { + case FILE_GPR: + a = v->reg.data.id; + b = a + v->reg.size / 4; + for (int r = a; r < b; ++r) + ready = MAX2(ready, score->wr.r[r]); + break; + case FILE_PREDICATE: + ready = MAX2(ready, score->wr.p[v->reg.data.id]); + break; + default: + assert(v->reg.file == FILE_FLAGS); + ready = MAX2(ready, score->wr.c); + break; + } + if (cycle < ready) + delay = MAX2(delay, ready - cycle); +} + +void +SchedDataCalculator::recordWr(const Value *v, const int ready) +{ + int a = v->reg.data.id; + + if (v->reg.file == FILE_GPR) { + int b = a + v->reg.size / 4; + for (int r = a; r < b; ++r) + score->rd.r[r] = ready; + } else + // $c, $pX: shorter issue-to-read delay (at least as exec pred and carry) + if (v->reg.file == FILE_PREDICATE) { + score->rd.p[a] = ready + 4; + } else { + assert(v->reg.file == FILE_FLAGS); + score->rd.c = ready + 4; + } +} + +void +SchedDataCalculator::recordRd(const Value *v, const int ready) +{ + int a = v->reg.data.id; + + if (v->reg.file == FILE_GPR) { + int b = a + v->reg.size / 4; + for (int r = a; r < b; ++r) + score->wr.r[r] = ready; + } else + if (v->reg.file == FILE_PREDICATE) { + score->wr.p[a] = ready; + } else + if (v->reg.file == FILE_FLAGS) { + score->wr.c = ready; + } +} + +void +CodeEmitterNVC0::prepareEmission(Function *func) +{ + const Target *targ = func->getProgram()->getTarget(); + + CodeEmitter::prepareEmission(func); + + if (targ->hasSWSched) { + SchedDataCalculator sched(targ); + sched.run(func, true, true); + } +} + +CodeEmitterNVC0::CodeEmitterNVC0(const TargetNVC0 *target) + : CodeEmitter(target), + writeIssueDelays(target->hasSWSched) { code = NULL; codeSize = codeSizeLimit = 0; diff --git a/src/gallium/drivers/nvc0/codegen/nv50_ir_target_nvc0.cpp b/src/gallium/drivers/nvc0/codegen/nv50_ir_target_nvc0.cpp index 93af23faac0..ffa40dd93fc 100644 --- a/src/gallium/drivers/nvc0/codegen/nv50_ir_target_nvc0.cpp +++ b/src/gallium/drivers/nvc0/codegen/nv50_ir_target_nvc0.cpp @@ -29,7 +29,7 @@ Target *getTargetNVC0(unsigned int chipset) return new TargetNVC0(chipset); } -TargetNVC0::TargetNVC0(unsigned int card) +TargetNVC0::TargetNVC0(unsigned int card) : Target(false, card >= 0xe4) { chipset = card; initOpInfo(); @@ -274,8 +274,6 @@ void TargetNVC0::initOpInfo() OP_QUADON, OP_QUADPOP, OP_TEXBAR }; - joinAnterior = false; - for (i = 0; i < DATA_FILE_COUNT; ++i) nativeFileMap[i] = (DataFile)i; nativeFileMap[FILE_ADDRESS] = FILE_GPR; @@ -534,14 +532,39 @@ TargetNVC0::isPostMultiplySupported(operation op, float f, int& e) const } // TODO: better values +// this could be more precise, e.g. depending on the issue-to-read/write delay +// of the depending instruction, but it's good enough int TargetNVC0::getLatency(const Instruction *i) const { - if (i->op == OP_LOAD) { - if (i->cache == CACHE_CV) - return 700; - return 48; + if (chipset >= 0xe4) { + if (i->dType == TYPE_F64 || i->sType == TYPE_F64) + return 20; + switch (i->op) { + case OP_LINTERP: + case OP_PINTERP: + return 15; + case OP_LOAD: + if (i->src(0).getFile() == FILE_MEMORY_CONST) + return 9; + // fall through + case OP_VFETCH: + return 24; + default: + if (Target::getOpClass(i->op) == OPCLASS_TEXTURE) + return 17; + if (i->op == OP_MUL && i->dType != TYPE_F32) + return 15; + return 9; + } + } else { + if (i->op == OP_LOAD) { + if (i->cache == CACHE_CV) + return 700; + return 48; + } + return 24; } - return 24; + return 32; } // These are "inverse" throughput values, i.e. the number of cycles required @@ -613,4 +636,42 @@ int TargetNVC0::getThroughput(const Instruction *i) const } } +bool TargetNVC0::canDualIssue(const Instruction *a, const Instruction *b) const +{ + const OpClass clA = operationClass[a->op]; + const OpClass clB = operationClass[b->op]; + + if (getChipset() >= 0xe4) { + // not texturing + // not if the 2nd instruction isn't necessarily executed + if (clA == OPCLASS_TEXTURE || clA == OPCLASS_FLOW) + return false; + // anything with MOV + if (a->op == OP_MOV || b->op == OP_MOV) + return true; + if (clA == clB) { + // only F32 arith or integer additions + if (clA != OPCLASS_ARITH) + return false; + return (a->dType == TYPE_F32 || a->op == OP_ADD || + b->dType == TYPE_F32 || b->op == OP_ADD); + } + // nothing with TEXBAR + if (a->op == OP_TEXBAR || b->op == OP_TEXBAR) + return false; + // no loads and stores accessing the the same space + if ((clA == OPCLASS_LOAD && clB == OPCLASS_STORE) || + (clB == OPCLASS_LOAD && clA == OPCLASS_STORE)) + if (a->src(0).getFile() == b->src(0).getFile()) + return false; + // no > 32-bit ops + if (typeSizeof(a->dType) > 4 || typeSizeof(b->dType) > 4 || + typeSizeof(a->sType) > 4 || typeSizeof(b->sType) > 4) + return false; + return true; + } else { + return false; // info not needed (yet) + } +} + } // namespace nv50_ir diff --git a/src/gallium/drivers/nvc0/codegen/nv50_ir_target_nvc0.h b/src/gallium/drivers/nvc0/codegen/nv50_ir_target_nvc0.h index e4efe476de0..d859388dfdf 100644 --- a/src/gallium/drivers/nvc0/codegen/nv50_ir_target_nvc0.h +++ b/src/gallium/drivers/nvc0/codegen/nv50_ir_target_nvc0.h @@ -51,6 +51,7 @@ public: virtual bool isPostMultiplySupported(operation, float, int& e) const; virtual bool mayPredicate(const Instruction *, const Value *) const; + virtual bool canDualIssue(const Instruction *, const Instruction *) const; virtual int getLatency(const Instruction *) const; virtual int getThroughput(const Instruction *) const; |