swr: [rasterizer core] buckets fixes

1. Don't clear bucket descriptions to fix issues with sim level
   buckets getting out of sync.

2. Close out threadviz file descriptors in ClearThreads().

3. Skip buckets for jitter based buckets when multithreaded. We need
   thread local storage through llvm jit functions to be fixed before
   we can enable this.

4. Fix buckets StopCapture to correctly detect capture complete.

Reviewed-by: Bruce Cherniak <[email protected]>

5 files changed, 49 insertions, 25 deletions

diff --git a/src/gallium/drivers/swr/rasterizer/common/rdtsc_buckets.cpp b/src/gallium/drivers/swr/rasterizer/common/rdtsc_buckets.cpp
index 8df5deb3416..412182f22aa 100644
--- a/src/gallium/drivers/swr/rasterizer/common/rdtsc_buckets.cpp
+++ b/src/gallium/drivers/swr/rasterizer/common/rdtsc_buckets.cpp
@@ -175,6 +175,7 @@ void BucketManager::DumpThreadViz()
     {
         fflush(thread.vizFile);
         fclose(thread.vizFile);
+        thread.vizFile = nullptr;
     }
     mThreadMutex.unlock();
 
diff --git a/src/gallium/drivers/swr/rasterizer/common/rdtsc_buckets.h b/src/gallium/drivers/swr/rasterizer/common/rdtsc_buckets.h
index 95841163155..fe25e77832e 100644
--- a/src/gallium/drivers/swr/rasterizer/common/rdtsc_buckets.h
+++ b/src/gallium/drivers/swr/rasterizer/common/rdtsc_buckets.h
@@ -53,6 +53,17 @@ public:
     void ClearThreads()
     {
         mThreadMutex.lock();
+        // close out the threadviz files if threadviz is enabled
+        if (KNOB_BUCKETS_ENABLE_THREADVIZ)
+        {
+            for (auto& thread : mThreads)
+            {
+                if (thread.vizFile != nullptr)
+                {
+                    fclose(thread.vizFile);
+                }
+            }
+        }
         mThreads.clear();
         mThreadMutex.unlock();
     }
@@ -99,7 +110,7 @@ public:
             stillCapturing = false;
             for (const BUCKET_THREAD& t : mThreads)
             {
-                if (t.pCurrent != &t.root)
+                if (t.level > 0)
                 {
                     stillCapturing = true;
                     continue;
diff --git a/src/gallium/drivers/swr/rasterizer/core/rdtsc_core.cpp b/src/gallium/drivers/swr/rasterizer/core/rdtsc_core.cpp
index 52c84ee2465..56eed25f959 100644
--- a/src/gallium/drivers/swr/rasterizer/core/rdtsc_core.cpp
+++ b/src/gallium/drivers/swr/rasterizer/core/rdtsc_core.cpp
@@ -93,3 +93,4 @@ std::vector<uint32_t> gBucketMap;
 BucketManager gBucketMgr;
 
 uint32_t gCurrentFrame = 0;
+bool gBucketsInitialized = false;
diff --git a/src/gallium/drivers/swr/rasterizer/core/rdtsc_core.h b/src/gallium/drivers/swr/rasterizer/core/rdtsc_core.h
index e1dde61b386..36f36ab1fc1 100644
--- a/src/gallium/drivers/swr/rasterizer/core/rdtsc_core.h
+++ b/src/gallium/drivers/swr/rasterizer/core/rdtsc_core.h
@@ -122,24 +122,25 @@ extern std::vector<uint32_t> gBucketMap;
 extern BucketManager gBucketMgr;
 extern BUCKET_DESC gCoreBuckets[];
 extern uint32_t gCurrentFrame;
+extern bool gBucketsInitialized;
 
 INLINE void rdtscReset()
 {
     gCurrentFrame = 0;
     gBucketMgr.ClearThreads();
-    gBucketMgr.ClearBuckets();
 }
 
 INLINE void rdtscInit(int threadId)
 {
     // register all the buckets once
-    if (threadId == 0)
+    if (!gBucketsInitialized && (threadId == 0))
     {
         gBucketMap.resize(NumBuckets);
         for (uint32_t i = 0; i < NumBuckets; ++i)
         {
             gBucketMap[i] = gBucketMgr.RegisterBucket(gCoreBuckets[i]);
         }
+        gBucketsInitialized = true;
     }
 
     std::string name = threadId == 0 ? "API" : "WORKER";
diff --git a/src/gallium/drivers/swr/rasterizer/jitter/builder_misc.cpp b/src/gallium/drivers/swr/rasterizer/jitter/builder_misc.cpp
index 7da4c025a7a..2f4fa382ebf 100644
--- a/src/gallium/drivers/swr/rasterizer/jitter/builder_misc.cpp
+++ b/src/gallium/drivers/swr/rasterizer/jitter/builder_misc.cpp
@@ -1459,35 +1459,45 @@ Value *Builder::VINSERTI128(Value* a, Value* b, Constant* imm8)
 // rdtsc buckets macros
 void Builder::RDTSC_START(Value* pBucketMgr, Value* pId)
 {
-    std::vector<Type*> args{
-        PointerType::get(mInt32Ty, 0),   // pBucketMgr
-        mInt32Ty                        // id
-    };
-
-    FunctionType* pFuncTy = FunctionType::get(Type::getVoidTy(JM()->mContext), args, false);
-    Function* pFunc = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("BucketManager_StartBucket", pFuncTy));
-    if (sys::DynamicLibrary::SearchForAddressOfSymbol("BucketManager_StartBucket") == nullptr)
+    // @todo due to an issue with thread local storage propagation in llvm, we can only safely call into
+    // buckets framework when single threaded
+    if (KNOB_SINGLE_THREADED)
     {
-        sys::DynamicLibrary::AddSymbol("BucketManager_StartBucket", (void*)&BucketManager_StartBucket);
-    }
+        std::vector<Type*> args{
+            PointerType::get(mInt32Ty, 0),   // pBucketMgr
+            mInt32Ty                        // id
+        };
+
+        FunctionType* pFuncTy = FunctionType::get(Type::getVoidTy(JM()->mContext), args, false);
+        Function* pFunc = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("BucketManager_StartBucket", pFuncTy));
+        if (sys::DynamicLibrary::SearchForAddressOfSymbol("BucketManager_StartBucket") == nullptr)
+        {
+            sys::DynamicLibrary::AddSymbol("BucketManager_StartBucket", (void*)&BucketManager_StartBucket);
+        }
 
-    CALL(pFunc, { pBucketMgr, pId });
+        CALL(pFunc, { pBucketMgr, pId });
+    }
 }
 
 void Builder::RDTSC_STOP(Value* pBucketMgr, Value* pId)
 {
-    std::vector<Type*> args{
-        PointerType::get(mInt32Ty, 0),   // pBucketMgr
-        mInt32Ty                        // id
-    };
-
-    FunctionType* pFuncTy = FunctionType::get(Type::getVoidTy(JM()->mContext), args, false);
-    Function* pFunc = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("BucketManager_StopBucket", pFuncTy));
-    if (sys::DynamicLibrary::SearchForAddressOfSymbol("BucketManager_StopBucket") == nullptr)
+    // @todo due to an issue with thread local storage propagation in llvm, we can only safely call into
+    // buckets framework when single threaded
+    if (KNOB_SINGLE_THREADED)
     {
-        sys::DynamicLibrary::AddSymbol("BucketManager_StopBucket", (void*)&BucketManager_StopBucket);
-    }
+        std::vector<Type*> args{
+            PointerType::get(mInt32Ty, 0),   // pBucketMgr
+            mInt32Ty                        // id
+        };
+
+        FunctionType* pFuncTy = FunctionType::get(Type::getVoidTy(JM()->mContext), args, false);
+        Function* pFunc = cast<Function>(JM()->mpCurrentModule->getOrInsertFunction("BucketManager_StopBucket", pFuncTy));
+        if (sys::DynamicLibrary::SearchForAddressOfSymbol("BucketManager_StopBucket") == nullptr)
+        {
+            sys::DynamicLibrary::AddSymbol("BucketManager_StopBucket", (void*)&BucketManager_StopBucket);
+        }
 
-    CALL(pFunc, { pBucketMgr, pId });
+        CALL(pFunc, { pBucketMgr, pId });
+    }
 }