diff options
| -rw-r--r-- | src/gallium/drivers/swr/rasterizer/core/context.h | 6 | ||||
| -rw-r--r-- | src/gallium/drivers/swr/rasterizer/core/frontend.cpp | 31 | ||||
| -rw-r--r-- | src/gallium/drivers/swr/rasterizer/core/pa.h | 522 | ||||
| -rw-r--r-- | src/gallium/drivers/swr/rasterizer/core/pa_avx.cpp | 65 |
4 files changed, 331 insertions, 293 deletions
diff --git a/src/gallium/drivers/swr/rasterizer/core/context.h b/src/gallium/drivers/swr/rasterizer/core/context.h index 6e596b4900d..e937a631b9e 100644 --- a/src/gallium/drivers/swr/rasterizer/core/context.h +++ b/src/gallium/drivers/swr/rasterizer/core/context.h @@ -217,6 +217,12 @@ struct PA_STATE; typedef void(*PFN_PROCESS_PRIMS)(DRAW_CONTEXT *pDC, PA_STATE& pa, uint32_t workerId, simdvector prims[], uint32_t primMask, simdscalari primID, simdscalari viewportIdx); +#if ENABLE_AVX512_SIMD16 +// function signature for pipeline stages that execute after primitive assembly +typedef void(*PFN_PROCESS_PRIMS_SIMD16)(DRAW_CONTEXT *pDC, PA_STATE& pa, uint32_t workerId, simd16vector prims[], + uint32_t primMask, simd16scalari primID, simd16scalari viewportIdx); + +#endif OSALIGNLINE(struct) API_STATE { // Vertex Buffers diff --git a/src/gallium/drivers/swr/rasterizer/core/frontend.cpp b/src/gallium/drivers/swr/rasterizer/core/frontend.cpp index 4d04d8ace5d..c150c51199f 100644 --- a/src/gallium/drivers/swr/rasterizer/core/frontend.cpp +++ b/src/gallium/drivers/swr/rasterizer/core/frontend.cpp @@ -1295,7 +1295,7 @@ void ProcessDraw( while (pa.HasWork()) { - // PaGetNextVsOutput currently has the side effect of updating some PA state machine state. + // GetNextVsOutput currently has the side effect of updating some PA state machine state. // So we need to keep this outside of (i < endVertex) check. simdmask *pvCutIndices_lo = nullptr; @@ -1303,8 +1303,10 @@ void ProcessDraw( if (IsIndexedT::value) { - pvCutIndices_lo = &pa.GetNextVsIndices(); - pvCutIndices_hi = &pa.GetNextVsIndices(); + // simd16mask <=> simdmask[2] + + pvCutIndices_lo = &reinterpret_cast<simdmask *>(&pa.GetNextVsIndices())[0]; + pvCutIndices_hi = &reinterpret_cast<simdmask *>(&pa.GetNextVsIndices())[1]; } simdvertex vout_lo; @@ -1313,7 +1315,7 @@ void ProcessDraw( vsContext_lo.pVout = &vout_lo; vsContext_hi.pVout = &vout_hi; - simd16vertex &vout = pa.GetNextVsOutput_simd16(); + simd16vertex &vout = pa.GetNextVsOutput(); if (i < endVertex) { @@ -1433,12 +1435,13 @@ void ProcessDraw( { SWR_ASSERT(pDC->pState->pfnProcessPrims); - uint32_t genMask = GenMask(pa.NumPrims_simd16()); - uint32_t genMask_lo = genMask & 255; - uint32_t genMask_hi = (genMask >> 8) & 255; + uint32_t mask = GenMask(pa.NumPrims()); + uint32_t mask_lo = mask & 255; + uint32_t mask_hi = (mask >> 8) & 255; - simdscalari getPrimId_lo = pa.GetPrimID_simd16_lo(work.startPrimID); - simdscalari getPrimId_hi = pa.GetPrimID_simd16_hi(work.startPrimID); + simd16scalari primid = pa.GetPrimID(work.startPrimID); + simdscalari primid_lo = primid.lo; + simdscalari primid_hi = primid.hi; simdvector prim[MAX_NUM_VERTS_PER_PRIM]; @@ -1451,10 +1454,9 @@ void ProcessDraw( } pa.useAlternateOffset = false; - pDC->pState->pfnProcessPrims(pDC, pa, workerId, prim, - genMask_lo, getPrimId_lo, _simd_set1_epi32(0)); + pDC->pState->pfnProcessPrims(pDC, pa, workerId, prim, mask_lo, primid_lo, _simd_setzero_si()); - if (genMask_hi) + if (mask_hi) { for (uint32_t i = 0; i < 3; i += 1) { @@ -1465,8 +1467,7 @@ void ProcessDraw( } pa.useAlternateOffset = true; - pDC->pState->pfnProcessPrims(pDC, pa, workerId, prim, - genMask_hi, getPrimId_hi, _simd_set1_epi32(0)); + pDC->pState->pfnProcessPrims(pDC, pa, workerId, prim, mask_hi, primid_hi, _simd_setzero_si()); } } } @@ -1543,7 +1544,7 @@ void ProcessDraw( while (pa.HasWork()) { - // PaGetNextVsOutput currently has the side effect of updating some PA state machine state. + // GetNextVsOutput currently has the side effect of updating some PA state machine state. // So we need to keep this outside of (i < endVertex) check. simdmask* pvCutIndices = nullptr; if (IsIndexedT::value) diff --git a/src/gallium/drivers/swr/rasterizer/core/pa.h b/src/gallium/drivers/swr/rasterizer/core/pa.h index 7319c56b4d5..7f60a04a9ff 100644 --- a/src/gallium/drivers/swr/rasterizer/core/pa.h +++ b/src/gallium/drivers/swr/rasterizer/core/pa.h @@ -34,6 +34,39 @@ struct PA_STATE { +#if USE_SIMD16_FRONTEND + enum + { + SIMD_WIDTH = KNOB_SIMD16_WIDTH, + SIMD_WIDTH_DIV2 = KNOB_SIMD16_WIDTH / 2, + SIMD_WIDTH_LOG2 = 4 + }; + + typedef simd16mask SIMDMASK; + + typedef simd16scalar SIMDSCALAR; + typedef simd16vector SIMDVECTOR; + typedef simd16vertex SIMDVERTEX; + + typedef simd16scalari SIMDSCALARI; + +#else + enum + { + SIMD_WIDTH = KNOB_SIMD_WIDTH, + SIMD_WIDTH_DIV2 = KNOB_SIMD_WIDTH / 2, + SIMD_WIDTH_LOG2 = 3 + }; + + typedef simdmask SIMDMASK; + + typedef simdscalar SIMDSCALAR; + typedef simdvector SIMDVECTOR; + typedef simdvertex SIMDVERTEX; + + typedef simdscalari SIMDSCALARI; + +#endif DRAW_CONTEXT *pDC{ nullptr }; // draw context uint8_t* pStreamBase{ nullptr }; // vertex stream uint32_t streamSizeInVerts{ 0 }; // total size of the input stream in verts @@ -60,24 +93,12 @@ struct PA_STATE #endif virtual void AssembleSingle(uint32_t slot, uint32_t primIndex, __m128 verts[]) = 0; virtual bool NextPrim() = 0; - virtual simdvertex& GetNextVsOutput() = 0; -#if ENABLE_AVX512_SIMD16 - virtual simdvertex& GetNextVsOutput_simd16_lo() = 0; - virtual simdvertex& GetNextVsOutput_simd16_hi() = 0; - virtual simd16vertex& GetNextVsOutput_simd16() = 0; -#endif + virtual SIMDVERTEX& GetNextVsOutput() = 0; virtual bool GetNextStreamOutput() = 0; - virtual simdmask& GetNextVsIndices() = 0; + virtual SIMDMASK& GetNextVsIndices() = 0; virtual uint32_t NumPrims() = 0; -#if ENABLE_AVX512_SIMD16 - virtual uint32_t NumPrims_simd16() = 0; -#endif virtual void Reset() = 0; - virtual simdscalari GetPrimID(uint32_t startID) = 0; -#if ENABLE_AVX512_SIMD16 - virtual simdscalari GetPrimID_simd16_lo(uint32_t startID) = 0; - virtual simdscalari GetPrimID_simd16_hi(uint32_t startID) = 0; -#endif + virtual SIMDSCALARI GetPrimID(uint32_t startID) = 0; }; // The Optimized PA is a state machine that assembles triangles from vertex shader simd @@ -98,7 +119,8 @@ struct PA_STATE // cuts struct PA_STATE_OPT : public PA_STATE { - simdvertex leadingVertex; // For tri-fan + SIMDVERTEX leadingVertex; // For tri-fan + uint32_t numPrims{ 0 }; // Total number of primitives for draw. uint32_t numPrimsComplete{ 0 }; // Total number of complete primitives. @@ -112,20 +134,22 @@ struct PA_STATE_OPT : public PA_STATE bool reset{ false }; // reset state uint32_t primIDIncr{ 0 }; // how much to increment for each vector (typically vector / {1, 2}) - simdscalari primID; + SIMDSCALARI primID; typedef bool(*PFN_PA_FUNC)(PA_STATE_OPT& state, uint32_t slot, simdvector verts[]); - typedef void(*PFN_PA_SINGLE_FUNC)(PA_STATE_OPT& pa, uint32_t slot, uint32_t primIndex, __m128 verts[]); #if ENABLE_AVX512_SIMD16 typedef bool(*PFN_PA_FUNC_SIMD16)(PA_STATE_OPT& state, uint32_t slot, simd16vector verts[]); #endif + typedef void(*PFN_PA_SINGLE_FUNC)(PA_STATE_OPT& pa, uint32_t slot, uint32_t primIndex, __m128 verts[]); PFN_PA_FUNC pfnPaFunc{ nullptr }; // PA state machine function for assembling 4 triangles. +#if ENABLE_AVX512_SIMD16 + PFN_PA_FUNC_SIMD16 pfnPaFunc_simd16{ nullptr }; +#endif PFN_PA_SINGLE_FUNC pfnPaSingleFunc{ nullptr }; // PA state machine function for assembling single triangle. PFN_PA_FUNC pfnPaFuncReset{ nullptr }; // initial state to set on reset #if ENABLE_AVX512_SIMD16 - PFN_PA_FUNC_SIMD16 pfnPaFunc_simd16{ nullptr }; // PA state machine function for assembling 16 triangles - PFN_PA_FUNC_SIMD16 pfnPaFuncReset_simd16{ nullptr }; // initial state to set on reset + PFN_PA_FUNC_SIMD16 pfnPaFuncReset_simd16{ nullptr }; #endif // state used to advance the PA when Next is called @@ -138,7 +162,7 @@ struct PA_STATE_OPT : public PA_STATE bool nextReset{ false }; bool isStreaming{ false }; - simdmask tmpIndices{ 0 }; // temporary index store for unused virtual function + SIMDMASK tmpIndices{ 0 }; // temporary index store for unused virtual function PA_STATE_OPT() {} PA_STATE_OPT(DRAW_CONTEXT* pDC, uint32_t numPrims, uint8_t* pStream, uint32_t streamSizeInVerts, @@ -221,55 +245,18 @@ struct PA_STATE_OPT : public PA_STATE return morePrims; } - simdvertex& GetNextVsOutput() + SIMDVERTEX& GetNextVsOutput() { // increment cur and prev indices - const uint32_t numSimdVerts = this->streamSizeInVerts / KNOB_SIMD_WIDTH; + const uint32_t numSimdVerts = this->streamSizeInVerts / SIMD_WIDTH; this->prev = this->cur; // prev is undefined for first state. this->cur = this->counter % numSimdVerts; - simdvertex* pVertex = (simdvertex*)pStreamBase; + SIMDVERTEX* pVertex = (SIMDVERTEX*)pStreamBase; return pVertex[this->cur]; } -#if ENABLE_AVX512_SIMD16 - simdvertex& GetNextVsOutput_simd16_lo() - { - // increment cur and prev indices - const uint32_t numSimdVerts = this->streamSizeInVerts / KNOB_SIMD16_WIDTH; - this->prev = this->cur; // prev is undefined for first state. - this->cur = this->counter % numSimdVerts; - - simdvertex* pVertex = (simdvertex*)pStreamBase; - return pVertex[this->cur * 2]; - } - - simdvertex& GetNextVsOutput_simd16_hi() - { - // increment cur and prev indices - const uint32_t numSimdVerts = this->streamSizeInVerts / KNOB_SIMD16_WIDTH; -#if 1 - this->prev = this->cur; // prev is undefined for first state. - this->cur = this->counter % numSimdVerts; -#endif - - simdvertex* pVertex = (simdvertex*)pStreamBase; - return pVertex[this->cur * 2 + 1]; - } - - simd16vertex& GetNextVsOutput_simd16() - { - // increment cur and prev indices - const uint32_t numSimdVerts = this->streamSizeInVerts / KNOB_SIMD16_WIDTH; - this->prev = this->cur; // prev is undefined for first state. - this->cur = this->counter % numSimdVerts; - - simd16vertex* pVertex = (simd16vertex*)pStreamBase; - return pVertex[this->cur]; - } - -#endif - simdmask& GetNextVsIndices() + SIMDMASK& GetNextVsIndices() { // unused in optimized PA, pass tmp buffer back return tmpIndices; @@ -286,17 +273,9 @@ struct PA_STATE_OPT : public PA_STATE uint32_t NumPrims() { return (this->numPrimsComplete + this->nextNumPrimsIncrement > this->numPrims) ? - (KNOB_SIMD_WIDTH - (this->numPrimsComplete + this->nextNumPrimsIncrement - this->numPrims)) : KNOB_SIMD_WIDTH; + (SIMD_WIDTH - (this->numPrimsComplete + this->nextNumPrimsIncrement - this->numPrims)) : SIMD_WIDTH; } -#if ENABLE_AVX512_SIMD16 - uint32_t NumPrims_simd16() - { - return (this->numPrimsComplete + this->nextNumPrimsIncrement > this->numPrims) ? - (KNOB_SIMD16_WIDTH - (this->numPrimsComplete + this->nextNumPrimsIncrement - this->numPrims)) : KNOB_SIMD16_WIDTH; - } - -#endif void SetNextState(PA_STATE_OPT::PFN_PA_FUNC pfnPaNextFunc, PA_STATE_OPT::PFN_PA_SINGLE_FUNC pfnPaNextSingleFunc, uint32_t numSimdPrims = 0, @@ -343,33 +322,16 @@ struct PA_STATE_OPT : public PA_STATE this->reset = false; } - simdscalari GetPrimID(uint32_t startID) + SIMDSCALARI GetPrimID(uint32_t startID) { - return _simd_add_epi32(this->primID, - _simd_set1_epi32(startID + this->primIDIncr * (this->numPrimsComplete / KNOB_SIMD_WIDTH))); - } -#if ENABLE_AVX512_SIMD16 - - simdscalari GetPrimID_simd16_lo(uint32_t startID) - { -#if 1 - return _simd_add_epi32(this->primID, - _simd_set1_epi32(startID + (this->primIDIncr / 2) * (this->numPrimsComplete / KNOB_SIMD_WIDTH) * 2)); +#if USE_SIMD16_FRONTEND + return _simd16_add_epi32(this->primID, + _simd16_set1_epi32(startID + this->primIDIncr * (this->numPrimsComplete / SIMD_WIDTH))); #else - return _simd_set1_epi32(0); -#endif - } - - simdscalari GetPrimID_simd16_hi(uint32_t startID) - { -#if 1 return _simd_add_epi32(this->primID, - _simd_set1_epi32(startID + (this->primIDIncr / 2) * ((this->numPrimsComplete / KNOB_SIMD_WIDTH) * 2 + 1))); -#else - return _simd_set1_epi32(0); + _simd_set1_epi32(startID + this->primIDIncr * (this->numPrimsComplete / SIMD_WIDTH))); #endif } -#endif }; // helper C wrappers to avoid having to rewrite all the PA topology state functions @@ -489,22 +451,26 @@ INLINE __m128 swizzleLaneN(const simdvector &a, int lane) // Cut-aware primitive assembler. struct PA_STATE_CUT : public PA_STATE { - simdmask* pCutIndices{ nullptr }; // cut indices buffer, 1 bit per vertex + SIMDMASK* pCutIndices{ nullptr }; // cut indices buffer, 1 bit per vertex uint32_t numVerts{ 0 }; // number of vertices available in buffer store uint32_t numAttribs{ 0 }; // number of attributes int32_t numRemainingVerts{ 0 }; // number of verts remaining to be assembled uint32_t numVertsToAssemble{ 0 }; // total number of verts to assemble for the draw - OSALIGNSIMD(uint32_t) indices[MAX_NUM_VERTS_PER_PRIM][KNOB_SIMD_WIDTH]; // current index buffer for gather - simdscalari vOffsets[MAX_NUM_VERTS_PER_PRIM]; // byte offsets for currently assembling simd +#if ENABLE_AVX512_SIMD16 + OSALIGNSIMD16(uint32_t) indices[MAX_NUM_VERTS_PER_PRIM][SIMD_WIDTH]; // current index buffer for gather +#else + OSALIGNSIMD(uint32_t) indices[MAX_NUM_VERTS_PER_PRIM][SIMD_WIDTH]; // current index buffer for gather +#endif + SIMDSCALARI vOffsets[MAX_NUM_VERTS_PER_PRIM]; // byte offsets for currently assembling simd uint32_t numPrimsAssembled{ 0 }; // number of primitives that are fully assembled uint32_t headVertex{ 0 }; // current unused vertex slot in vertex buffer store uint32_t tailVertex{ 0 }; // beginning vertex currently assembling uint32_t curVertex{ 0 }; // current unprocessed vertex uint32_t startPrimId{ 0 }; // starting prim id - simdscalari vPrimId; // vector of prim ID + SIMDSCALARI vPrimId; // vector of prim ID bool needOffsets{ false }; // need to compute gather offsets for current SIMD uint32_t vertsPerPrim{ 0 }; - simdvertex tmpVertex; // temporary simdvertex for unimplemented API + SIMDVERTEX tmpVertex; // temporary simdvertex for unimplemented API bool processCutVerts{ false }; // vertex indices with cuts should be processed as normal, otherwise they // are ignored. Fetch shader sends invalid verts on cuts that should be ignored // while the GS sends valid verts for every index @@ -518,7 +484,7 @@ struct PA_STATE_CUT : public PA_STATE PFN_PA_FUNC pfnPa{ nullptr }; // per-topology function that processes a single vert PA_STATE_CUT() {} - PA_STATE_CUT(DRAW_CONTEXT* pDC, uint8_t* in_pStream, uint32_t in_streamSizeInVerts, simdmask* in_pIndices, uint32_t in_numVerts, + PA_STATE_CUT(DRAW_CONTEXT* pDC, uint8_t* in_pStream, uint32_t in_streamSizeInVerts, SIMDMASK* in_pIndices, uint32_t in_numVerts, uint32_t in_numAttribs, PRIMITIVE_TOPOLOGY topo, bool in_processCutVerts) : PA_STATE(pDC, in_pStream, in_streamSizeInVerts) { @@ -535,7 +501,11 @@ struct PA_STATE_CUT : public PA_STATE curIndex = 0; pCutIndices = in_pIndices; memset(indices, 0, sizeof(indices)); - vPrimId = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); +#if USE_SIMD16_FRONTEND + vPrimId = _simd16_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); +#else + vPrimId = _simd_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); +#endif reverseWinding = false; adjExtraVert = -1; @@ -566,44 +536,18 @@ struct PA_STATE_CUT : public PA_STATE } } - simdvertex& GetNextVsOutput() - { - uint32_t vertexIndex = this->headVertex / KNOB_SIMD_WIDTH; - this->headVertex = (this->headVertex + KNOB_SIMD_WIDTH) % this->numVerts; - this->needOffsets = true; - return ((simdvertex*)pStreamBase)[vertexIndex]; - } - -#if ENABLE_AVX512_SIMD16 - simdvertex& GetNextVsOutput_simd16_lo() - { - uint32_t vertexIndex = this->headVertex / KNOB_SIMD16_WIDTH; - this->headVertex = (this->headVertex + KNOB_SIMD16_WIDTH) % this->numVerts; - this->needOffsets = true; - return ((simdvertex*)pStreamBase)[vertexIndex * 2]; - } - - simdvertex& GetNextVsOutput_simd16_hi() - { - uint32_t vertexIndex = this->headVertex / KNOB_SIMD16_WIDTH; - this->headVertex = (this->headVertex + KNOB_SIMD16_WIDTH) % this->numVerts; - this->needOffsets = true; - return ((simdvertex*)pStreamBase)[vertexIndex * 2 + 1]; - } - - simd16vertex& GetNextVsOutput_simd16() + SIMDVERTEX& GetNextVsOutput() { - uint32_t vertexIndex = this->headVertex / KNOB_SIMD16_WIDTH; - this->headVertex = (this->headVertex + KNOB_SIMD16_WIDTH) % this->numVerts; + uint32_t vertexIndex = this->headVertex / SIMD_WIDTH; + this->headVertex = (this->headVertex + SIMD_WIDTH) % this->numVerts; this->needOffsets = true; - return ((simd16vertex*)pStreamBase)[vertexIndex]; + return ((SIMDVERTEX*)pStreamBase)[vertexIndex]; } -#endif - simdmask& GetNextVsIndices() + SIMDMASK& GetNextVsIndices() { - uint32_t vertexIndex = this->headVertex / KNOB_SIMD_WIDTH; - simdmask* pCurCutIndex = this->pCutIndices + vertexIndex; + uint32_t vertexIndex = this->headVertex / SIMD_WIDTH; + SIMDMASK* pCurCutIndex = this->pCutIndices + vertexIndex; return *pCurCutIndex; } @@ -611,7 +555,8 @@ struct PA_STATE_CUT : public PA_STATE { // unused SWR_ASSERT(0 && "Not implemented"); - return this->tmpVertex.attrib[0]; + static simdvector junk; + return junk; } #if ENABLE_AVX512_SIMD16 @@ -626,28 +571,20 @@ struct PA_STATE_CUT : public PA_STATE #endif bool GetNextStreamOutput() { - this->headVertex += KNOB_SIMD_WIDTH; + this->headVertex += SIMD_WIDTH; this->needOffsets = true; return HasWork(); } - simdscalari GetPrimID(uint32_t startID) - { - return _simd_add_epi32(_simd_set1_epi32(startID), this->vPrimId); - } - -#if ENABLE_AVX512_SIMD16 - simdscalari GetPrimID_simd16_lo(uint32_t startID) + SIMDSCALARI GetPrimID(uint32_t startID) { +#if USE_SIMD16_FRONTEND + return _simd16_add_epi32(_simd16_set1_epi32(startID), this->vPrimId); +#else return _simd_add_epi32(_simd_set1_epi32(startID), this->vPrimId); +#endif } - simdscalari GetPrimID_simd16_hi(uint32_t startID) - { - return _simd_add_epi32(_simd_set1_epi32(startID + KNOB_SIMD_WIDTH), this->vPrimId); - } - -#endif void Reset() { #if ENABLE_AVX512_SIMD16 @@ -662,7 +599,11 @@ struct PA_STATE_CUT : public PA_STATE this->headVertex = 0; this->reverseWinding = false; this->adjExtraVert = -1; - this->vPrimId = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); +#if USE_SIMD16_FRONTEND + this->vPrimId = _simd16_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); +#else + this->vPrimId = _simd_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); +#endif } bool HasWork() @@ -672,7 +613,7 @@ struct PA_STATE_CUT : public PA_STATE bool IsVertexStoreFull() { - return ((this->headVertex + KNOB_SIMD_WIDTH) % this->numVerts) == this->tailVertex; + return ((this->headVertex + SIMD_WIDTH) % this->numVerts) == this->tailVertex; } void RestartTopology() @@ -684,8 +625,8 @@ struct PA_STATE_CUT : public PA_STATE bool IsCutIndex(uint32_t vertex) { - uint32_t vertexIndex = vertex / KNOB_SIMD_WIDTH; - uint32_t vertexOffset = vertex & (KNOB_SIMD_WIDTH - 1); + uint32_t vertexIndex = vertex / SIMD_WIDTH; + uint32_t vertexOffset = vertex & (SIMD_WIDTH - 1); return _bittest((const LONG*)&this->pCutIndices[vertexIndex], vertexOffset) == 1; } @@ -693,7 +634,7 @@ struct PA_STATE_CUT : public PA_STATE // have assembled SIMD prims void ProcessVerts() { - while (this->numPrimsAssembled != KNOB_SIMD_WIDTH && + while (this->numPrimsAssembled != SIMD_WIDTH && this->numRemainingVerts > 0 && this->curVertex != this->headVertex) { @@ -724,7 +665,7 @@ struct PA_STATE_CUT : public PA_STATE } // special case last primitive for tri strip w/ adj - if (this->numPrimsAssembled != KNOB_SIMD_WIDTH && this->numRemainingVerts == 0 && this->adjExtraVert != -1) + if (this->numPrimsAssembled != SIMD_WIDTH && this->numRemainingVerts == 0 && this->adjExtraVert != -1) { (this->*pfnPa)(this->curVertex, true); } @@ -736,13 +677,17 @@ struct PA_STATE_CUT : public PA_STATE // advance tail to the current unsubmitted vertex this->tailVertex = this->curVertex; this->numPrimsAssembled = 0; - this->vPrimId = _simd_add_epi32(vPrimId, _simd_set1_epi32(KNOB_SIMD_WIDTH)); +#if USE_SIMD16_FRONTEND + this->vPrimId = _simd16_add_epi32(vPrimId, _simd16_set1_epi32(SIMD_WIDTH)); +#else + this->vPrimId = _simd_add_epi32(vPrimId, _simd_set1_epi32(SIMD_WIDTH)); +#endif } bool NextPrim() { // if we've assembled enough prims, we can advance to the next set of verts - if (this->numPrimsAssembled == KNOB_SIMD_WIDTH || this->numRemainingVerts <= 0) + if (this->numPrimsAssembled == SIMD_WIDTH || this->numRemainingVerts <= 0) { Advance(); } @@ -753,27 +698,37 @@ struct PA_STATE_CUT : public PA_STATE { for (uint32_t v = 0; v < this->vertsPerPrim; ++v) { - simdscalari vIndices = *(simdscalari*)&this->indices[v][0]; + SIMDSCALARI vIndices = *(SIMDSCALARI*)&this->indices[v][0]; // step to simdvertex batch - const uint32_t simdShift = 3; // @todo make knob - simdscalari vVertexBatch = _simd_srai_epi32(vIndices, simdShift); - this->vOffsets[v] = _simd_mullo_epi32(vVertexBatch, _simd_set1_epi32(sizeof(simdvertex))); + const uint32_t simdShift = SIMD_WIDTH_LOG2; +#if USE_SIMD16_FRONTEND + SIMDSCALARI vVertexBatch = _simd16_srai_epi32(vIndices, simdShift); + this->vOffsets[v] = _simd16_mullo_epi32(vVertexBatch, _simd16_set1_epi32(sizeof(SIMDVERTEX))); +#else + SIMDSCALARI vVertexBatch = _simd_srai_epi32(vIndices, simdShift); + this->vOffsets[v] = _simd_mullo_epi32(vVertexBatch, _simd_set1_epi32(sizeof(SIMDVERTEX))); +#endif // step to index - const uint32_t simdMask = 0x7; // @todo make knob - simdscalari vVertexIndex = _simd_and_si(vIndices, _simd_set1_epi32(simdMask)); + const uint32_t simdMask = SIMD_WIDTH - 1; +#if USE_SIMD16_FRONTEND + SIMDSCALARI vVertexIndex = _simd16_and_si(vIndices, _simd16_set1_epi32(simdMask)); + this->vOffsets[v] = _simd16_add_epi32(this->vOffsets[v], _simd16_mullo_epi32(vVertexIndex, _simd16_set1_epi32(sizeof(float)))); +#else + SIMDSCALARI vVertexIndex = _simd_and_si(vIndices, _simd_set1_epi32(simdMask)); this->vOffsets[v] = _simd_add_epi32(this->vOffsets[v], _simd_mullo_epi32(vVertexIndex, _simd_set1_epi32(sizeof(float)))); +#endif } } - bool Assemble(uint32_t slot, simdvector result[]) + bool Assemble(uint32_t slot, simdvector verts[]) { // process any outstanding verts ProcessVerts(); // return false if we don't have enough prims assembled - if (this->numPrimsAssembled != KNOB_SIMD_WIDTH && this->numRemainingVerts > 0) + if (this->numPrimsAssembled != SIMD_WIDTH && this->numRemainingVerts > 0) { return false; } @@ -787,18 +742,28 @@ struct PA_STATE_CUT : public PA_STATE for (uint32_t v = 0; v < this->vertsPerPrim; ++v) { - simdscalari offsets = this->vOffsets[v]; + SIMDSCALARI offsets = this->vOffsets[v]; // step to attribute - offsets = _simd_add_epi32(offsets, _simd_set1_epi32(slot * sizeof(simdvector))); +#if USE_SIMD16_FRONTEND + offsets = _simd16_add_epi32(offsets, _simd16_set1_epi32(slot * sizeof(SIMDVECTOR))); +#else + offsets = _simd_add_epi32(offsets, _simd_set1_epi32(slot * sizeof(SIMDVECTOR))); +#endif float* pBase = (float*)this->pStreamBase; for (uint32_t c = 0; c < 4; ++c) { - result[v].v[c] = _simd_i32gather_ps(pBase, offsets, 1); +#if USE_SIMD16_FRONTEND + simd16scalar temp = _simd16_i32gather_ps(pBase, offsets, 1); + + verts[v].v[c] = useAlternateOffset ? temp.hi : temp.lo; +#else + verts[v].v[c] = _simd_i32gather_ps(pBase, offsets, 1); +#endif // move base to next component - pBase += KNOB_SIMD_WIDTH; + pBase += SIMD_WIDTH; } } @@ -808,8 +773,49 @@ struct PA_STATE_CUT : public PA_STATE #if ENABLE_AVX512_SIMD16 bool Assemble_simd16(uint32_t slot, simd16vector verts[]) { - SWR_ASSERT(false); - return false; + // process any outstanding verts + ProcessVerts(); + + // return false if we don't have enough prims assembled + if (this->numPrimsAssembled != SIMD_WIDTH && this->numRemainingVerts > 0) + { + return false; + } + + // cache off gather offsets given the current SIMD set of indices the first time we get an assemble + if (this->needOffsets) + { + ComputeOffsets(); + this->needOffsets = false; + } + + for (uint32_t v = 0; v < this->vertsPerPrim; ++v) + { + SIMDSCALARI offsets = this->vOffsets[v]; + + // step to attribute +#if USE_SIMD16_FRONTEND + offsets = _simd16_add_epi32(offsets, _simd16_set1_epi32(slot * sizeof(SIMDVECTOR))); +#else + offsets = _simd_add_epi32(offsets, _simd_set1_epi32(slot * sizeof(simdvector))); +#endif + + float* pBase = (float*)this->pStreamBase; + for (uint32_t c = 0; c < 4; ++c) + { +#if USE_SIMD16_FRONTEND + verts[v].v[c] = _simd16_i32gather_ps(pBase, offsets, 1); +#else + verts[v].v[c].lo = _simd_i32gather_ps(pBase, offsets, 1); + verts[v].v[c].hi = _simd_setzero_ps(); +#endif + + // move base to next component + pBase += SIMD_WIDTH; + } + } + + return true; } #endif @@ -819,14 +825,18 @@ struct PA_STATE_CUT : public PA_STATE for (uint32_t v = 0; v < this->vertsPerPrim; ++v) { uint32_t* pOffset = (uint32_t*)&this->vOffsets[v]; +#if USE_SIMD16_FRONTEND + uint32_t offset = useAlternateOffset ? pOffset[triIndex + SIMD_WIDTH_DIV2] : pOffset[triIndex]; +#else uint32_t offset = pOffset[triIndex]; - offset += sizeof(simdvector) * slot; +#endif + offset += sizeof(SIMDVECTOR) * slot; float* pVert = (float*)&tri[v]; for (uint32_t c = 0; c < 4; ++c) { float* pComponent = (float*)(this->pStreamBase + offset); pVert[c] = *pComponent; - offset += KNOB_SIMD_WIDTH * sizeof(float); + offset += SIMD_WIDTH * sizeof(float); } } } @@ -836,13 +846,6 @@ struct PA_STATE_CUT : public PA_STATE return this->numPrimsAssembled; } -#if ENABLE_AVX512_SIMD16 - uint32_t NumPrims_simd16() - { - return this->numPrimsAssembled; - } - -#endif // Per-topology functions void ProcessVertTriStrip(uint32_t index, bool finish) { @@ -1188,7 +1191,7 @@ struct PA_TESS : PA_STATE { PA_TESS( DRAW_CONTEXT *in_pDC, - const simdscalar* in_pVertData, + const SIMDSCALAR* in_pVertData, uint32_t in_attributeStrideInVectors, uint32_t in_numAttributes, uint32_t* (&in_ppIndices)[3], @@ -1201,7 +1204,11 @@ struct PA_TESS : PA_STATE m_numAttributes(in_numAttributes), m_numPrims(in_numPrims) { +#if USE_SIMD16_FRONTEND + m_vPrimId = _simd16_setzero_si(); +#else m_vPrimId = _simd_setzero_si(); +#endif binTopology = in_binTopology; m_ppIndices[0] = in_ppIndices[0]; m_ppIndices[1] = in_ppIndices[1]; @@ -1248,40 +1255,30 @@ struct PA_TESS : PA_STATE } #endif - static simdscalari GenPrimMask(uint32_t numPrims) + static SIMDSCALARI GenPrimMask(uint32_t numPrims) { - SWR_ASSERT(numPrims <= KNOB_SIMD_WIDTH); -#if KNOB_SIMD_WIDTH == 8 - static const OSALIGNLINE(int32_t) maskGen[KNOB_SIMD_WIDTH * 2] = + SWR_ASSERT(numPrims <= SIMD_WIDTH); +#if USE_SIMD16_FRONTEND + static const OSALIGNLINE(int32_t) maskGen[SIMD_WIDTH * 2] = { - -1, -1, -1, -1, -1, -1, -1, -1, - 0, 0, 0, 0, 0, 0, 0, 0 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; -#else -#error "Help, help, I can't get up!" -#endif - - return _simd_loadu_si((const simdscalari*)&maskGen[KNOB_SIMD_WIDTH - numPrims]); - } - -#if ENABLE_AVX512_SIMD16 - static simd16scalari GenPrimMask_simd16(uint32_t numPrims) - { - SWR_ASSERT(numPrims <= KNOB_SIMD16_WIDTH); - static const OSALIGNSIMD16(int32_t) maskGen_16[KNOB_SIMD16_WIDTH * 2] = + return _simd16_loadu_si((const SIMDSCALARI*)&maskGen[SIMD_WIDTH - numPrims]); +#else + static const OSALIGNLINE(int32_t) maskGen[SIMD_WIDTH * 2] = { - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, 0, 0, 0 }; - return _simd16_loadu_si((const simd16scalari*)&maskGen_16[KNOB_SIMD16_WIDTH - numPrims]); + return _simd_loadu_si((const SIMDSCALARI*)&maskGen[SIMD_WIDTH - numPrims]); +#endif } -#endif bool Assemble(uint32_t slot, simdvector verts[]) { - static_assert(KNOB_SIMD_WIDTH == 8, "Need to revisit this when AVX512 is implemented"); SWR_ASSERT(slot < m_numAttributes); uint32_t numPrimsToAssemble = PA_TESS::NumPrims(); @@ -1290,23 +1287,38 @@ struct PA_TESS : PA_STATE return false; } - simdscalari mask = GenPrimMask(numPrimsToAssemble); + SIMDSCALARI mask = GenPrimMask(numPrimsToAssemble); const float* pBaseAttrib = (const float*)&m_pVertexData[slot * m_attributeStrideInVectors * 4]; for (uint32_t i = 0; i < m_numVertsPerPrim; ++i) { - simdscalari indices = _simd_load_si((const simdscalari*)m_ppIndices[i]); +#if USE_SIMD16_FRONTEND + SIMDSCALARI indices = _simd16_load_si((const SIMDSCALARI*)m_ppIndices[i]); +#else + SIMDSCALARI indices = _simd_load_si((const SIMDSCALARI*)m_ppIndices[i]); +#endif const float* pBase = pBaseAttrib; for (uint32_t c = 0; c < 4; ++c) { +#if USE_SIMD16_FRONTEND + simd16scalar temp = _simd16_mask_i32gather_ps( + _simd16_setzero_ps(), + pBase, + indices, + mask, + 4 /* gcc doesn't like sizeof(float) */); + + verts[i].v[c] = useAlternateOffset ? temp.hi : temp.lo; +#else verts[i].v[c] = _simd_mask_i32gather_ps( _simd_setzero_ps(), pBase, indices, _simd_castsi_ps(mask), 4 /* gcc doesn't like sizeof(float) */); - pBase += m_attributeStrideInVectors * KNOB_SIMD_WIDTH; +#endif + pBase += m_attributeStrideInVectors * SIMD_WIDTH; } } @@ -1318,29 +1330,43 @@ struct PA_TESS : PA_STATE { SWR_ASSERT(slot < m_numAttributes); - uint32_t numPrimsToAssemble = PA_TESS::NumPrims_simd16(); + uint32_t numPrimsToAssemble = PA_TESS::NumPrims(); if (0 == numPrimsToAssemble) { return false; } - simd16scalari mask = GenPrimMask_simd16(numPrimsToAssemble); + SIMDSCALARI mask = GenPrimMask(numPrimsToAssemble); const float* pBaseAttrib = (const float*)&m_pVertexData[slot * m_attributeStrideInVectors * 4]; for (uint32_t i = 0; i < m_numVertsPerPrim; ++i) { - simd16scalari indices = _simd16_load_si((const simd16scalari*)m_ppIndices[i]); +#if USE_SIMD16_FRONTEND + SIMDSCALARI indices = _simd16_load_si((const SIMDSCALARI*)m_ppIndices[i]); +#else + SIMDSCALARI indices = _simd_load_si((const SIMDSCALARI*)m_ppIndices[i]); +#endif const float* pBase = pBaseAttrib; for (uint32_t c = 0; c < 4; ++c) { +#if USE_SIMD16_FRONTEND verts[i].v[c] = _simd16_mask_i32gather_ps( _simd16_setzero_ps(), pBase, indices, mask, 4 /* gcc doesn't like sizeof(float) */); - pBase += m_attributeStrideInVectors * KNOB_SIMD16_WIDTH; +#else + verts[i].v[c].lo = _simd_mask_i32gather_ps( + _simd_setzero_ps(), + pBase, + indices, + _simd_castsi_ps(mask), + 4 /* gcc doesn't like sizeof(float) */); + verts[i].v[c].hi = _simd_setzero_ps(); +#endif + pBase += m_attributeStrideInVectors * SIMD_WIDTH; } } @@ -1356,14 +1382,18 @@ struct PA_TESS : PA_STATE const float* pVertDataBase = (const float*)&m_pVertexData[slot * m_attributeStrideInVectors * 4]; for (uint32_t i = 0; i < m_numVertsPerPrim; ++i) { +#if USE_SIMD16_FRONTEND + uint32_t index = useAlternateOffset ? m_ppIndices[i][primIndex + SIMD_WIDTH_DIV2] : m_ppIndices[i][primIndex]; +#else uint32_t index = m_ppIndices[i][primIndex]; +#endif const float* pVertData = pVertDataBase; float* pVert = (float*)&verts[i]; for (uint32_t c = 0; c < 4; ++c) { pVert[c] = pVertData[index]; - pVertData += m_attributeStrideInVectors * KNOB_SIMD_WIDTH; + pVertData += m_attributeStrideInVectors * SIMD_WIDTH; } } } @@ -1379,82 +1409,44 @@ struct PA_TESS : PA_STATE return HasWork(); } - simdvertex& GetNextVsOutput() - { - SWR_ASSERT(0, "%s", __FUNCTION__); - static simdvertex junk; - return junk; - } - -#if ENABLE_AVX512_SIMD16 - simdvertex& GetNextVsOutput_simd16_lo() - { - SWR_ASSERT(0, "%s", __FUNCTION__); - static simdvertex junk; - return junk; - } - - simdvertex& GetNextVsOutput_simd16_hi() + SIMDVERTEX& GetNextVsOutput() { SWR_ASSERT(0, "%s", __FUNCTION__); - static simdvertex junk; + static SIMDVERTEX junk; return junk; } - simd16vertex& GetNextVsOutput_simd16() - { - SWR_ASSERT(0, "%s", __FUNCTION__); - static simd16vertex junk; - return junk; - } - -#endif bool GetNextStreamOutput() { SWR_ASSERT(0, "%s", __FUNCTION__); return false; } - simdmask& GetNextVsIndices() + SIMDMASK& GetNextVsIndices() { SWR_ASSERT(0, "%s", __FUNCTION__); - static simdmask junk; + static SIMDMASK junk; return junk; } uint32_t NumPrims() { - return std::min<uint32_t>(m_numPrims, KNOB_SIMD_WIDTH); + return std::min<uint32_t>(m_numPrims, SIMD_WIDTH); } -#if ENABLE_AVX512_SIMD16 - uint32_t NumPrims_simd16() - { - return std::min<uint32_t>(m_numPrims, KNOB_SIMD16_WIDTH); - } - -#endif void Reset() { SWR_ASSERT(0); }; - simdscalari GetPrimID(uint32_t startID) - { - return _simd_add_epi32(_simd_set1_epi32(startID), m_vPrimId); - } - -#if ENABLE_AVX512_SIMD16 - simdscalari GetPrimID_simd16_lo(uint32_t startID) + SIMDSCALARI GetPrimID(uint32_t startID) { +#if USE_SIMD16_FRONTEND + return _simd16_add_epi32(_simd16_set1_epi32(startID), m_vPrimId); +#else return _simd_add_epi32(_simd_set1_epi32(startID), m_vPrimId); +#endif } - simdscalari GetPrimID_simd16_hi(uint32_t startID) - { - return _simd_add_epi32(_simd_set1_epi32(startID + KNOB_SIMD_WIDTH), m_vPrimId); - } - -#endif private: - const simdscalar* m_pVertexData = nullptr; + const SIMDSCALAR* m_pVertexData = nullptr; uint32_t m_attributeStrideInVectors = 0; uint32_t m_numAttributes = 0; uint32_t m_numPrims = 0; @@ -1462,7 +1454,7 @@ private: uint32_t m_numVertsPerPrim = 0; - simdscalari m_vPrimId; + SIMDSCALARI m_vPrimId; }; // Primitive Assembler factory class, responsible for creating and initializing the correct assembler @@ -1486,7 +1478,7 @@ struct PA_FACTORY memset(&indexStore, 0, sizeof(indexStore)); uint32_t numAttribs = state.feNumAttributes; - new (&this->paCut) PA_STATE_CUT(pDC, (uint8_t*)&this->vertexStore[0], MAX_NUM_VERTS_PER_PRIM * KNOB_SIMD_WIDTH, + new (&this->paCut) PA_STATE_CUT(pDC, (uint8_t*)&this->vertexStore[0], MAX_NUM_VERTS_PER_PRIM * PA_STATE::SIMD_WIDTH, &this->indexStore[0], numVerts, numAttribs, state.topology, false); cutPA = true; } @@ -1494,7 +1486,7 @@ struct PA_FACTORY #endif { uint32_t numPrims = GetNumPrims(in_topo, numVerts); - new (&this->paOpt) PA_STATE_OPT(pDC, numPrims, (uint8_t*)&this->vertexStore[0], MAX_NUM_VERTS_PER_PRIM * KNOB_SIMD_WIDTH, false); + new (&this->paOpt) PA_STATE_OPT(pDC, numPrims, (uint8_t*)&this->vertexStore[0], MAX_NUM_VERTS_PER_PRIM * PA_STATE::SIMD_WIDTH, false); cutPA = false; } @@ -1520,6 +1512,6 @@ struct PA_FACTORY PRIMITIVE_TOPOLOGY topo{ TOP_UNKNOWN }; - simdvertex vertexStore[MAX_NUM_VERTS_PER_PRIM]; - simdmask indexStore[MAX_NUM_VERTS_PER_PRIM]; + PA_STATE::SIMDVERTEX vertexStore[MAX_NUM_VERTS_PER_PRIM]; + PA_STATE::SIMDMASK indexStore[MAX_NUM_VERTS_PER_PRIM]; }; diff --git a/src/gallium/drivers/swr/rasterizer/core/pa_avx.cpp b/src/gallium/drivers/swr/rasterizer/core/pa_avx.cpp index eec824703df..a8d8379297e 100644 --- a/src/gallium/drivers/swr/rasterizer/core/pa_avx.cpp +++ b/src/gallium/drivers/swr/rasterizer/core/pa_avx.cpp @@ -245,6 +245,10 @@ bool PaTriList2(PA_STATE_OPT& pa, uint32_t slot, simdvector verts[]) #elif KNOB_ARCH >= KNOB_ARCH_AVX2 + const simdscalari perm0 = _simd_set_epi32(5, 2, 7, 4, 1, 6, 3, 0); + const simdscalari perm1 = _simd_set_epi32(6, 3, 0, 5, 2, 7, 4, 1); + const simdscalari perm2 = _simd_set_epi32(7, 4, 1, 6, 3, 0, 5, 2); + const simdvector &a = PaGetSimdVector(pa, 0, slot); const simdvector &b = PaGetSimdVector(pa, 1, slot); const simdvector &c = PaGetSimdVector(pa, 2, slot); @@ -253,10 +257,6 @@ bool PaTriList2(PA_STATE_OPT& pa, uint32_t slot, simdvector verts[]) // v1 -> a1 a4 a7 b2 b5 c0 c3 c6 // v2 -> a2 a5 b0 b3 b6 c1 c4 c7 - const simdscalari perm0 = _simd_set_epi32(5, 2, 7, 4, 1, 6, 3, 0); - const simdscalari perm1 = _simd_set_epi32(6, 3, 0, 5, 2, 7, 4, 1); - const simdscalari perm2 = _simd_set_epi32(7, 4, 1, 6, 3, 0, 5, 2); - simdvector &v0 = verts[0]; simdvector &v1 = verts[1]; simdvector &v2 = verts[2]; @@ -334,7 +334,7 @@ void PaTriListSingle0(PA_STATE_OPT& pa, uint32_t slot, uint32_t primIndex, __m12 // We have 12 simdscalars contained within 3 simdvectors which // hold at least 8 triangles worth of data. We want to assemble a single // triangle with data in horizontal form. -#if ENABLE_AVX512_SIMD16 +#if USE_SIMD16_FRONTEND const simd16vector &a_16 = PaGetSimdVector_simd16(pa, 0, slot); const simd16vector &b_16 = PaGetSimdVector_simd16(pa, 1, slot); const simd16vector &c_16 = PaGetSimdVector_simd16(pa, 2, slot); @@ -559,7 +559,7 @@ bool PaTriFan0(PA_STATE_OPT& pa, uint32_t slot, simdvector verts[]) } // store off leading vertex for attributes - simdvertex* pVertex = (simdvertex*)pa.pStreamBase; + PA_STATE_OPT::SIMDVERTEX* pVertex = (PA_STATE_OPT::SIMDVERTEX*)pa.pStreamBase; pa.leadingVertex = pVertex[pa.cur]; SetNextPaState(pa, PaTriFan1, PaTriFanSingle0); @@ -568,7 +568,7 @@ bool PaTriFan0(PA_STATE_OPT& pa, uint32_t slot, simdvector verts[]) bool PaTriFan1(PA_STATE_OPT& pa, uint32_t slot, simdvector verts[]) { - simdvector& leadVert = pa.leadingVertex.attrib[slot]; + PA_STATE_OPT::SIMDVECTOR& leadVert = pa.leadingVertex.attrib[slot]; simdvector& a = PaGetSimdVector(pa, pa.prev, slot); simdvector& b = PaGetSimdVector(pa, pa.cur, slot); simdscalar s; @@ -579,7 +579,11 @@ bool PaTriFan1(PA_STATE_OPT& pa, uint32_t slot, simdvector verts[]) simdscalar a0 = a[i]; simdscalar b0 = b[i]; +#if USE_SIMD16_FRONTEND + __m256 comp = leadVert[i].lo; +#else __m256 comp = leadVert[i]; +#endif simdvector& v0 = verts[0]; v0[i] = _simd_shuffle_ps(comp, comp, _MM_SHUFFLE(0, 0, 0, 0)); v0[i] = _mm256_permute2f128_ps(v0[i], comp, 0x00); @@ -599,8 +603,19 @@ bool PaTriFan1(PA_STATE_OPT& pa, uint32_t slot, simdvector verts[]) void PaTriFanSingle0(PA_STATE_OPT& pa, uint32_t slot, uint32_t primIndex, __m128 verts[]) { // vert 0 from leading vertex - simdvector& lead = pa.leadingVertex.attrib[slot]; +#if USE_SIMD16_FRONTEND + PA_STATE_OPT::SIMDVECTOR& temp = pa.leadingVertex.attrib[slot]; + + simdvector lead; + lead[0] = temp[0].lo; + lead[1] = temp[1].lo; + lead[2] = temp[2].lo; + lead[3] = temp[3].lo; verts[0] = swizzleLane0(lead); +#else + PA_STATE_OPT::SIMDVECTOR& lead = pa.leadingVertex.attrib[slot]; + verts[0] = swizzleLane0(lead); +#endif simdvector& a = PaGetSimdVector(pa, pa.prev, slot); simdvector& b = PaGetSimdVector(pa, pa.cur, slot); @@ -1201,7 +1216,7 @@ void PaRectListSingle0( // We have 12 simdscalars contained within 3 simdvectors which // hold at least 8 triangles worth of data. We want to assemble a single // triangle with data in horizontal form. -#if ENABLE_AVX512_SIMD16 +#if USE_SIMD16_FRONTEND const simd16vector &a_16 = PaGetSimdVector_simd16(pa, 0, slot); const simd16vector &b_16 = PaGetSimdVector_simd16(pa, 1, slot); @@ -1417,11 +1432,15 @@ PA_STATE_OPT::PA_STATE_OPT(DRAW_CONTEXT *in_pDC, uint32_t in_numPrims, uint8_t* this->pfnPaFuncReset_simd16 = this->pfnPaFunc_simd16; #endif - // simdscalari id8 = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); - // simdscalari id4 = _mm256_set_epi32(0, 0, 1, 1, 2, 2, 3, 3); - simdscalari id8 = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); - simdscalari id4 = _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0); +#if USE_SIMD16_FRONTEND + simd16scalari id16 = _simd16_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + simd16scalari id82 = _simd16_set_epi32( 7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0); + +#else + simdscalari id8 = _simd_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); + simdscalari id4 = _simd_set_epi32(3, 3, 2, 2, 1, 1, 0, 0); +#endif switch(this->binTopology) { case TOP_TRIANGLE_LIST: @@ -1430,18 +1449,33 @@ PA_STATE_OPT::PA_STATE_OPT(DRAW_CONTEXT *in_pDC, uint32_t in_numPrims, uint8_t* case TOP_LINE_STRIP: case TOP_LINE_LIST: case TOP_LINE_LOOP: +#if USE_SIMD16_FRONTEND + this->primIDIncr = 16; + this->primID = id16; +#else this->primIDIncr = 8; this->primID = id8; +#endif break; case TOP_QUAD_LIST: case TOP_QUAD_STRIP: case TOP_RECT_LIST: +#if USE_SIMD16_FRONTEND + this->primIDIncr = 8; + this->primID = id82; +#else this->primIDIncr = 4; this->primID = id4; +#endif break; case TOP_POINT_LIST: +#if USE_SIMD16_FRONTEND + this->primIDIncr = 16; + this->primID = id16; +#else this->primIDIncr = 8; this->primID = id8; +#endif break; case TOP_PATCHLIST_1: case TOP_PATCHLIST_2: @@ -1476,8 +1510,13 @@ PA_STATE_OPT::PA_STATE_OPT(DRAW_CONTEXT *in_pDC, uint32_t in_numPrims, uint8_t* case TOP_PATCHLIST_31: case TOP_PATCHLIST_32: // Always run KNOB_SIMD_WIDTH number of patches at a time. +#if USE_SIMD16_FRONTEND + this->primIDIncr = 16; + this->primID = id16; +#else this->primIDIncr = 8; this->primID = id8; +#endif break; default: |