diff options
Diffstat (limited to 'src')
4 files changed, 220 insertions, 29 deletions
diff --git a/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_ir_macros.py b/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_ir_macros.py index ce892a9abe8..44fc8573719 100644 --- a/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_ir_macros.py +++ b/src/gallium/drivers/swr/rasterizer/codegen/gen_llvm_ir_macros.py @@ -44,6 +44,7 @@ inst_aliases = { intrinsics = [ ['VGATHERPD', 'x86_avx2_gather_d_pd_256', ['src', 'pBase', 'indices', 'mask', 'scale']], ['VGATHERPS', 'x86_avx2_gather_d_ps_256', ['src', 'pBase', 'indices', 'mask', 'scale']], + ['VGATHERPS2', 'x86_avx512_gather_dps_512', ['src', 'pBase', 'indices', 'mask', 'scale']], ['VGATHERDD', 'x86_avx2_gather_d_d_256', ['src', 'pBase', 'indices', 'mask', 'scale']], ['VPSRLI', 'x86_avx2_psrli_d', ['src', 'imm']], ['VSQRTPS', 'x86_avx_sqrt_ps_256', ['a']], diff --git a/src/gallium/drivers/swr/rasterizer/jitter/builder_misc.cpp b/src/gallium/drivers/swr/rasterizer/jitter/builder_misc.cpp index bd3a52566d6..8ffe05b41c3 100644 --- a/src/gallium/drivers/swr/rasterizer/jitter/builder_misc.cpp +++ b/src/gallium/drivers/swr/rasterizer/jitter/builder_misc.cpp @@ -211,6 +211,28 @@ namespace SwrJit return ConstantVector::getSplat(mVWidth, cast<ConstantInt>(C(i))); } +#if USE_SIMD16_BUILDER + Value *Builder::VIMMED2_1(int i) + { + return ConstantVector::getSplat(mVWidth2, cast<ConstantInt>(C(i))); + } + + Value *Builder::VIMMED2_1(uint32_t i) + { + return ConstantVector::getSplat(mVWidth2, cast<ConstantInt>(C(i))); + } + + Value *Builder::VIMMED2_1(float i) + { + return ConstantVector::getSplat(mVWidth2, cast<ConstantFP>(C(i))); + } + + Value *Builder::VIMMED2_1(bool i) + { + return ConstantVector::getSplat(mVWidth2, cast<ConstantInt>(C(i))); + } + +#endif Value *Builder::VUNDEF_IPTR() { return UndefValue::get(VectorType::get(mInt32PtrTy,mVWidth)); @@ -237,6 +259,11 @@ namespace SwrJit return UndefValue::get(VectorType::get(mFP32Ty, mVWidth2)); } + Value *Builder::VUNDEF2_I() + { + return UndefValue::get(VectorType::get(mInt32Ty, mVWidth2)); + } + #endif Value *Builder::VUNDEF(Type* t) { @@ -254,6 +281,19 @@ namespace SwrJit return VECTOR_SPLAT(mVWidth, src); } +#if USE_SIMD16_BUILDER + Value *Builder::VBROADCAST2(Value *src) + { + // check if src is already a vector + if (src->getType()->isVectorTy()) + { + return src; + } + + return VECTOR_SPLAT(mVWidth2, src); + } + +#endif uint32_t Builder::IMMED(Value* v) { SWR_ASSERT(isa<ConstantInt>(v)); @@ -554,16 +594,17 @@ namespace SwrJit /// @param vIndices - SIMD wide value of VB byte offsets /// @param vMask - SIMD wide mask that controls whether to access memory or the src values /// @param scale - value to scale indices by - Value *Builder::GATHERPS(Value* vSrc, Value* pBase, Value* vIndices, Value* vMask, uint8_t scale) + Value *Builder::GATHERPS(Value *vSrc, Value *pBase, Value *vIndices, Value *vMask, uint8_t scale) { - Value* vGather; + Value *vGather; // use avx2 gather instruction if available if(JM()->mArch.AVX2()) { // force mask to <N x float>, required by vgather - vMask = BITCAST(vMask, mSimdFP32Ty); - vGather = VGATHERPS(vSrc,pBase,vIndices,vMask,C(scale)); + Value *mask = BITCAST(vMask, mSimdFP32Ty); + + vGather = VGATHERPS(vSrc, pBase, vIndices, mask, C(scale)); } else { @@ -598,6 +639,41 @@ namespace SwrJit return vGather; } +#if USE_SIMD16_BUILDER + Value *Builder::GATHERPS2(Value *vSrc, Value *pBase, Value *vIndices, Value *vMask, uint8_t scale) + { + Value *vGather = VUNDEF2_F(); + + // use avx512 gather instruction if available + if (JM()->mArch.AVX512F()) + { + // force mask to <N-bit Integer>, required by vgather2 + Value *mask = BITCAST(MASK2(vMask), mInt16Ty); + + vGather = VGATHERPS2(vSrc, pBase, vIndices, mask, C((uint32_t)scale)); + } + else + { + Value *src0 = EXTRACT2_F(vSrc, 0); + Value *src1 = EXTRACT2_F(vSrc, 1); + + Value *indices0 = EXTRACT2_I(vIndices, 0); + Value *indices1 = EXTRACT2_I(vIndices, 1); + + Value *mask0 = EXTRACT2_I(vMask, 0); + Value *mask1 = EXTRACT2_I(vMask, 1); + + Value *gather0 = GATHERPS(src0, pBase, indices0, mask0, scale); + Value *gather1 = GATHERPS(src1, pBase, indices1, mask1, scale); + + vGather = INSERT2_F(vGather, gather0, 0); + vGather = INSERT2_F(vGather, gather1, 1); + } + + return vGather; + } + +#endif ////////////////////////////////////////////////////////////////////////// /// @brief Generate a masked gather operation in LLVM IR. If not /// supported on the underlying platform, emulate it with loads @@ -700,7 +776,7 @@ namespace SwrJit #if USE_SIMD16_BUILDER ////////////////////////////////////////////////////////////////////////// /// @brief - Value *Builder::EXTRACT(Value *a2, uint32_t imm) + Value *Builder::EXTRACT2_F(Value *a2, uint32_t imm) { const uint32_t i0 = (imm > 0) ? mVWidth : 0; @@ -708,6 +784,13 @@ namespace SwrJit for (uint32_t i = 0; i < mVWidth; i += 1) { +#if 1 + if (!a2->getType()->getScalarType()->isFloatTy()) + { + a2 = BITCAST(a2, mSimd2FP32Ty); + } + +#endif Value *temp = VEXTRACT(a2, C(i0 + i)); result = VINSERT(result, temp, C(i)); @@ -716,9 +799,14 @@ namespace SwrJit return result; } + Value *Builder::EXTRACT2_I(Value *a2, uint32_t imm) + { + return BITCAST(EXTRACT2_F(a2, imm), mSimdInt32Ty); + } + ////////////////////////////////////////////////////////////////////////// /// @brief - Value *Builder::INSERT(Value *a2, Value * b, uint32_t imm) + Value *Builder::INSERT2_F(Value *a2, Value *b, uint32_t imm) { const uint32_t i0 = (imm > 0) ? mVWidth : 0; @@ -741,22 +829,42 @@ namespace SwrJit return result; } + Value *Builder::INSERT2_I(Value *a2, Value *b, uint32_t imm) + { + return BITCAST(INSERT2_F(a2, b, imm), mSimd2Int32Ty); + } + #endif ////////////////////////////////////////////////////////////////////////// /// @brief convert x86 <N x float> mask to llvm <N x i1> mask - Value* Builder::MASK(Value* vmask) + Value *Builder::MASK(Value *vmask) { - Value* src = BITCAST(vmask, mSimdInt32Ty); + Value *src = BITCAST(vmask, mSimdInt32Ty); return ICMP_SLT(src, VIMMED1(0)); } +#if USE_SIMD16_BUILDER + Value *Builder::MASK2(Value *vmask) + { + Value *src = BITCAST(vmask, mSimd2Int32Ty); + return ICMP_SLT(src, VIMMED2_1(0)); + } + +#endif ////////////////////////////////////////////////////////////////////////// /// @brief convert llvm <N x i1> mask to x86 <N x i32> mask - Value* Builder::VMASK(Value* mask) + Value *Builder::VMASK(Value *mask) { return S_EXT(mask, mSimdInt32Ty); } +#if USE_SIMD16_BUILDER + Value *Builder::VMASK2(Value *mask) + { + return S_EXT(mask, mSimd2Int32Ty); + } + +#endif ////////////////////////////////////////////////////////////////////////// /// @brief Generate a VPSHUFB operation in LLVM IR. If not /// supported on the underlying platform, emulate it diff --git a/src/gallium/drivers/swr/rasterizer/jitter/builder_misc.h b/src/gallium/drivers/swr/rasterizer/jitter/builder_misc.h index 9aa24148d38..d858a827db2 100644 --- a/src/gallium/drivers/swr/rasterizer/jitter/builder_misc.h +++ b/src/gallium/drivers/swr/rasterizer/jitter/builder_misc.h @@ -54,15 +54,25 @@ Value *VIMMED1(int i); Value *VIMMED1(uint32_t i); Value *VIMMED1(float i); Value *VIMMED1(bool i); +#if USE_SIMD16_BUILDER +Value *VIMMED2_1(int i); +Value *VIMMED2_1(uint32_t i); +Value *VIMMED2_1(float i); +Value *VIMMED2_1(bool i); +#endif Value *VUNDEF(Type* t); Value *VUNDEF_F(); +Value *VUNDEF_I(); #if USE_SIMD16_BUILDER Value *VUNDEF2_F(); +Value *VUNDEF2_I(); #endif -Value *VUNDEF_I(); Value *VUNDEF(Type* ty, uint32_t size); Value *VUNDEF_IPTR(); Value *VBROADCAST(Value *src); +#if USE_SIMD16_BUILDER +Value *VBROADCAST2(Value *src); +#endif Value *VRCP(Value *va); Value *VPLANEPS(Value* vA, Value* vB, Value* vC, Value* &vX, Value* &vY); @@ -94,8 +104,12 @@ Value *VCMPPS_GE(Value* a, Value* b) { return VCMPPS(a, b, C((uint8_t)_CMP_GE Value *VCMPPS_GT(Value* a, Value* b) { return VCMPPS(a, b, C((uint8_t)_CMP_GT_OQ)); } Value *VCMPPS_NOTNAN(Value* a, Value* b){ return VCMPPS(a, b, C((uint8_t)_CMP_ORD_Q)); } -Value *MASK(Value* vmask); -Value *VMASK(Value* mask); +Value *MASK(Value *vmask); +Value *VMASK(Value *mask); +#if USE_SIMD16_BUILDER +Value *MASK2(Value *vmask); +Value *VMASK2(Value *mask); +#endif ////////////////////////////////////////////////////////////////////////// /// @brief functions that build IR to call x86 intrinsics directly, or @@ -103,8 +117,10 @@ Value *VMASK(Value* mask); ////////////////////////////////////////////////////////////////////////// #if USE_SIMD16_BUILDER -Value *EXTRACT(Value *a, uint32_t imm); -Value *INSERT(Value *a, Value *b, uint32_t imm); +Value *EXTRACT2_F(Value *a2, uint32_t imm); +Value *EXTRACT2_I(Value *a2, uint32_t imm); +Value *INSERT2_F(Value *a2, Value *b, uint32_t imm); +Value *INSERT2_I(Value *a2, Value *b, uint32_t imm); #endif Value *MASKLOADD(Value* src, Value* mask); @@ -112,7 +128,10 @@ Value *MASKLOADD(Value* src, Value* mask); void Gather4(const SWR_FORMAT format, Value* pSrcBase, Value* byteOffsets, Value* mask, Value* vGatherComponents[], bool bPackedOutput); -Value *GATHERPS(Value* src, Value* pBase, Value* indices, Value* mask, uint8_t scale = 1); +Value *GATHERPS(Value *src, Value *pBase, Value *indices, Value *mask, uint8_t scale = 1); +#if USE_SIMD16_BUILDER +Value *GATHERPS2(Value *src, Value *pBase, Value *indices, Value *mask, uint8_t scale = 1); +#endif void GATHER4PS(const SWR_FORMAT_INFO &info, Value* pSrcBase, Value* byteOffsets, Value* mask, Value* vGatherComponents[], bool bPackedOutput); diff --git a/src/gallium/drivers/swr/rasterizer/jitter/fetch_jit.cpp b/src/gallium/drivers/swr/rasterizer/jitter/fetch_jit.cpp index aa3fca4c358..d409792c32e 100644 --- a/src/gallium/drivers/swr/rasterizer/jitter/fetch_jit.cpp +++ b/src/gallium/drivers/swr/rasterizer/jitter/fetch_jit.cpp @@ -89,6 +89,9 @@ struct FetchJit : public Builder #else Value* GenerateCompCtrlVector(const ComponentControl ctrl); #endif +#if USE_SIMD16_BUILDER + Value* GenerateCompCtrlVector2(const ComponentControl ctrl); +#endif void JitLoadVertices(const FETCH_COMPILE_STATE &fetchState, Value* streams, Value* vIndices, Value* pVtxOut); #if USE_SIMD16_SHADERS @@ -1219,6 +1222,12 @@ void FetchJit::JitGatherVertices(const FETCH_COMPILE_STATE &fetchState, break; case 32: { +#if USE_SIMD16_GATHERS +#if USE_SIMD16_BUILDER + Value *pVtxSrc2[4]; + +#endif +#endif for (uint32_t i = 0; i < 4; i += 1) { #if USE_SIMD16_GATHERS @@ -1228,7 +1237,7 @@ void FetchJit::JitGatherVertices(const FETCH_COMPILE_STATE &fetchState, if (compCtrl[i] == StoreSrc) { // save mask as it is zero'd out after each gather - Value *vMask = vGatherMask; + Value *vMask = vGatherMask; Value *vMask2 = vGatherMask2; // Gather a SIMD of vertices @@ -1236,37 +1245,66 @@ void FetchJit::JitGatherVertices(const FETCH_COMPILE_STATE &fetchState, // However, GATHERPS uses signed 32-bit offsets, so only a 2GB range :( // But, we know that elements must be aligned for FETCH. :) // Right shift the offset by a bit and then scale by 2 to remove the sign extension. - Value *vShiftedOffsets = VPSRLI(vOffsets, C(1)); + Value *vShiftedOffsets = VPSRLI(vOffsets, C(1)); Value *vShiftedOffsets2 = VPSRLI(vOffsets2, C(1)); - vVertexElements[currentVertexElement] = GATHERPS(gatherSrc, pStreamBase, vShiftedOffsets, vMask, 2); +#if USE_SIMD16_BUILDER + Value *src = VUNDEF2_F(); + src = INSERT2_F(src, gatherSrc, 0); + src = INSERT2_F(src, gatherSrc2, 1); + + Value *indices = VUNDEF2_I(); + indices = INSERT2_I(indices, vShiftedOffsets, 0); + indices = INSERT2_I(indices, vShiftedOffsets2, 1); + + Value *mask = VUNDEF2_I(); + mask = INSERT2_I(mask, vMask, 0); + mask = INSERT2_I(mask, vMask2, 1); + + pVtxSrc2[currentVertexElement] = GATHERPS2(src, pStreamBase, indices, mask, 2); +#if 1 + + vVertexElements[currentVertexElement] = EXTRACT2_F(pVtxSrc2[currentVertexElement], 0); + vVertexElements2[currentVertexElement] = EXTRACT2_F(pVtxSrc2[currentVertexElement], 1); +#endif +#else + vVertexElements[currentVertexElement] = GATHERPS(gatherSrc, pStreamBase, vShiftedOffsets, vMask, 2); vVertexElements2[currentVertexElement] = GATHERPS(gatherSrc2, pStreamBase, vShiftedOffsets2, vMask2, 2); +#if USE_SIMD16_BUILDER + // pack adjacent pairs of SIMD8s into SIMD16s + pVtxSrc2[currentVertexElement] = VUNDEF2_F(); + pVtxSrc2[currentVertexElement] = INSERT2_F(pVtxSrc2[currentVertexElement], vVertexElements[currentVertexElement], 0); + pVtxSrc2[currentVertexElement] = INSERT2_F(pVtxSrc2[currentVertexElement], vVertexElements2[currentVertexElement], 1); + +#endif +#endif currentVertexElement += 1; } else { +#if USE_SIMD16_BUILDER + pVtxSrc2[currentVertexElement] = GenerateCompCtrlVector2(compCtrl[i]); +#else vVertexElements[currentVertexElement] = GenerateCompCtrlVector(compCtrl[i], false); vVertexElements2[currentVertexElement] = GenerateCompCtrlVector(compCtrl[i], true); +#if USE_SIMD16_BUILDER + // pack adjacent pairs of SIMD8s into SIMD16s + pVtxSrc2[currentVertexElement] = VUNDEF2_F(); + pVtxSrc2[currentVertexElement] = INSERT2_F(pVtxSrc2[currentVertexElement], vVertexElements[currentVertexElement], 0); + pVtxSrc2[currentVertexElement] = INSERT2_F(pVtxSrc2[currentVertexElement], vVertexElements2[currentVertexElement], 1); + +#endif +#endif currentVertexElement += 1; } if (currentVertexElement > 3) { #if USE_SIMD16_BUILDER - Value *pVtxSrc2[4]; - - // pack adjacent pairs of SIMD8s into SIMD16s - for (uint32_t i = 0; i < 4; i += 1) - { - pVtxSrc2[i] = VUNDEF2_F(); - - pVtxSrc2[i] = INSERT(pVtxSrc2[i], vVertexElements[i], 0); - pVtxSrc2[i] = INSERT(pVtxSrc2[i], vVertexElements2[i], 1); - } - // store SIMD16s Value *pVtxOut2 = BITCAST(pVtxOut, PointerType::get(VectorType::get(mFP32Ty, mVWidth2), 0)); + StoreVertexElements2(pVtxOut2, outputElt, 4, pVtxSrc2); #else @@ -2429,6 +2467,31 @@ Value* FetchJit::GenerateCompCtrlVector(const ComponentControl ctrl) } } +#if USE_SIMD16_BUILDER +Value* FetchJit::GenerateCompCtrlVector2(const ComponentControl ctrl) +{ + switch (ctrl) + { + case NoStore: return VUNDEF2_I(); + case Store0: return VIMMED2_1(0); + case Store1Fp: return VIMMED2_1(1.0f); + case Store1Int: return VIMMED2_1(1); + case StoreVertexId: + { + Value* pId = BITCAST(LOAD(GEP(mpFetchInfo, { 0, SWR_FETCH_CONTEXT_VertexID })), mSimd2FP32Ty); + return VBROADCAST2(pId); + } + case StoreInstanceId: + { + Value* pId = BITCAST(LOAD(GEP(mpFetchInfo, { 0, SWR_FETCH_CONTEXT_CurInstance })), mFP32Ty); + return VBROADCAST2(pId); + } + case StoreSrc: + default: SWR_INVALID("Invalid component control"); return VUNDEF2_I(); + } +} + +#endif ////////////////////////////////////////////////////////////////////////// /// @brief Returns the enable mask for the specified component. /// @param enableMask - enable bits |