clover: implement cl_arm_shared_virtual_memory

v2: use static array to keep name -> func mapping
v3: use unordered_map
v4: handle ARM constants
    reorder dispatch table
    wrap enqueue APIs as the command value differs between khr and arm
v5: move declarations into dispatch.hpp
    handle CL_MEM_USES_SVM_POINTER_ARM in clGetMemObjectInfo
v6: breaking long lines

Signed-off-by: Karol Herbst <[email protected]>
Reviewed-by: Francisco Jerez <[email protected]>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/2076>

6 files changed, 287 insertions, 54 deletions

diff --git a/src/gallium/state_trackers/clover/api/device.cpp b/src/gallium/state_trackers/clover/api/device.cpp
index d1c37b96aeb..042f2eda21c 100644
--- a/src/gallium/state_trackers/clover/api/device.cpp
+++ b/src/gallium/state_trackers/clover/api/device.cpp
@@ -406,6 +406,7 @@ clGetDeviceInfo(cl_device_id d_dev, cl_device_info param,
       break;
 
    case CL_DEVICE_SVM_CAPABILITIES:
+   case CL_DEVICE_SVM_CAPABILITIES_ARM:
       buf.as_scalar<cl_device_svm_capabilities>() = dev.svm_support();
       break;
 
diff --git a/src/gallium/state_trackers/clover/api/dispatch.hpp b/src/gallium/state_trackers/clover/api/dispatch.hpp
index ae93273f563..ea835ed6da4 100644
--- a/src/gallium/state_trackers/clover/api/dispatch.hpp
+++ b/src/gallium/state_trackers/clover/api/dispatch.hpp
@@ -46,6 +46,60 @@ namespace clover {
    cl_int
    IcdGetPlatformIDsKHR(cl_uint num_entries, cl_platform_id *rd_platforms,
                         cl_uint *rnum_platforms);
+
+   cl_int
+   EnqueueSVMFree(cl_command_queue command_queue,
+                  cl_uint num_svm_pointers,
+                  void *svm_pointers[],
+                  void (CL_CALLBACK *pfn_free_func) (
+                     cl_command_queue queue, cl_uint num_svm_pointers,
+                     void *svm_pointers[], void *user_data),
+                  void *user_data,
+                  cl_uint num_events_in_wait_list,
+                  const cl_event *event_wait_list,
+                  cl_event *event,
+                  cl_int cmd);
+
+   cl_int
+   EnqueueSVMMemcpy(cl_command_queue command_queue,
+                    cl_bool blocking_copy,
+                    void *dst_ptr,
+                    const void *src_ptr,
+                    size_t size,
+                    cl_uint num_events_in_wait_list,
+                    const cl_event *event_wait_list,
+                    cl_event *event,
+                    cl_int cmd);
+
+   cl_int
+   EnqueueSVMMap(cl_command_queue command_queue,
+                 cl_bool blocking_map,
+                 cl_map_flags map_flags,
+                 void *svm_ptr,
+                 size_t size,
+                 cl_uint num_events_in_wait_list,
+                 const cl_event *event_wait_list,
+                 cl_event *event,
+                 cl_int cmd);
+
+   cl_int
+   EnqueueSVMMemFill(cl_command_queue command_queue,
+                     void *svm_ptr,
+                     const void *pattern,
+                     size_t pattern_size,
+                     size_t size,
+                     cl_uint num_events_in_wait_list,
+                     const cl_event *event_wait_list,
+                     cl_event *event,
+                     cl_int cmd);
+
+   cl_int
+   EnqueueSVMUnmap(cl_command_queue command_queue,
+                   void *svm_ptr,
+                   cl_uint num_events_in_wait_list,
+                   const cl_event *event_wait_list,
+                   cl_event *event,
+                   cl_int cmd);
 }
 
 #endif
diff --git a/src/gallium/state_trackers/clover/api/kernel.cpp b/src/gallium/state_trackers/clover/api/kernel.cpp
index 962a2b59e16..31a87b63868 100644
--- a/src/gallium/state_trackers/clover/api/kernel.cpp
+++ b/src/gallium/state_trackers/clover/api/kernel.cpp
@@ -361,7 +361,8 @@ clSetKernelExecInfo(cl_kernel d_kern,
       return CL_INVALID_VALUE;
 
    switch (param_name) {
-   case CL_KERNEL_EXEC_INFO_SVM_FINE_GRAIN_SYSTEM: {
+   case CL_KERNEL_EXEC_INFO_SVM_FINE_GRAIN_SYSTEM:
+   case CL_KERNEL_EXEC_INFO_SVM_FINE_GRAIN_SYSTEM_ARM: {
       if (param_value_size != sizeof(cl_bool))
          return CL_INVALID_VALUE;
 
@@ -373,6 +374,7 @@ clSetKernelExecInfo(cl_kernel d_kern,
    }
 
    case CL_KERNEL_EXEC_INFO_SVM_PTRS:
+   case CL_KERNEL_EXEC_INFO_SVM_PTRS_ARM:
       if (has_system_svm)
          return CL_SUCCESS;
 
diff --git a/src/gallium/state_trackers/clover/api/memory.cpp b/src/gallium/state_trackers/clover/api/memory.cpp
index 107815bde72..e03793339c1 100644
--- a/src/gallium/state_trackers/clover/api/memory.cpp
+++ b/src/gallium/state_trackers/clover/api/memory.cpp
@@ -322,7 +322,8 @@ clGetMemObjectInfo(cl_mem d_mem, cl_mem_info param,
       buf.as_scalar<size_t>() = (sub ? sub->offset() : 0);
       break;
    }
-   case CL_MEM_USES_SVM_POINTER: {
+   case CL_MEM_USES_SVM_POINTER:
+   case CL_MEM_USES_SVM_POINTER_ARM: {
       // with system SVM all host ptrs are SVM pointers
       // TODO: once we support devices with lower levels of SVM, we have to
       // check the ptr in more detail
diff --git a/src/gallium/state_trackers/clover/api/platform.cpp b/src/gallium/state_trackers/clover/api/platform.cpp
index a684d34ba44..7360461e62f 100644
--- a/src/gallium/state_trackers/clover/api/platform.cpp
+++ b/src/gallium/state_trackers/clover/api/platform.cpp
@@ -20,6 +20,9 @@
 // OTHER DEALINGS IN THE SOFTWARE.
 //
 
+#include <unordered_map>
+
+#include "api/dispatch.hpp"
 #include "api/util.hpp"
 #include "core/platform.hpp"
 #include "git_sha1.h"
@@ -101,14 +104,105 @@ clover::GetExtensionFunctionAddressForPlatform(cl_platform_id d_platform,
    return NULL;
 }
 
-void *
-clover::GetExtensionFunctionAddress(const char *p_name) {
-   std::string name { p_name };
+namespace {
+
+cl_int
+enqueueSVMFreeARM(cl_command_queue command_queue,
+                  cl_uint num_svm_pointers,
+                  void *svm_pointers[],
+                  void (CL_CALLBACK *pfn_free_func) (
+                    cl_command_queue queue, cl_uint num_svm_pointers,
+                    void *svm_pointers[], void *user_data),
+                  void *user_data,
+                  cl_uint num_events_in_wait_list,
+                  const cl_event *event_wait_list,
+                  cl_event *event) {
+
+   return EnqueueSVMFree(command_queue, num_svm_pointers, svm_pointers,
+                         pfn_free_func, user_data, num_events_in_wait_list,
+                         event_wait_list, event, CL_COMMAND_SVM_FREE_ARM);
+}
+
+cl_int
+enqueueSVMMapARM(cl_command_queue command_queue,
+                 cl_bool blocking_map,
+                 cl_map_flags map_flags,
+                 void *svm_ptr,
+                 size_t size,
+                 cl_uint num_events_in_wait_list,
+                 const cl_event *event_wait_list,
+                 cl_event *event) {
+
+   return EnqueueSVMMap(command_queue, blocking_map, map_flags, svm_ptr, size,
+                        num_events_in_wait_list, event_wait_list, event,
+                        CL_COMMAND_SVM_MAP_ARM);
+}
+
+cl_int
+enqueueSVMMemcpyARM(cl_command_queue command_queue,
+                    cl_bool blocking_copy,
+                    void *dst_ptr,
+                    const void *src_ptr,
+                    size_t size,
+                    cl_uint num_events_in_wait_list,
+                    const cl_event *event_wait_list,
+                    cl_event *event) {
+
+   return EnqueueSVMMemcpy(command_queue, blocking_copy, dst_ptr, src_ptr,
+                           size, num_events_in_wait_list, event_wait_list,
+                           event, CL_COMMAND_SVM_MEMCPY_ARM);
+}
+
+cl_int
+enqueueSVMMemFillARM(cl_command_queue command_queue,
+                     void *svm_ptr,
+                     const void *pattern,
+                     size_t pattern_size,
+                     size_t size,
+                     cl_uint num_events_in_wait_list,
+                     const cl_event *event_wait_list,
+                     cl_event *event) {
+
+   return EnqueueSVMMemFill(command_queue, svm_ptr, pattern, pattern_size,
+                            size, num_events_in_wait_list, event_wait_list,
+                            event, CL_COMMAND_SVM_MEMFILL_ARM);
+}
 
-   if (name == "clIcdGetPlatformIDsKHR")
-      return reinterpret_cast<void *>(IcdGetPlatformIDsKHR);
-   else
-      return NULL;
+cl_int
+enqueueSVMUnmapARM(cl_command_queue command_queue,
+                   void *svm_ptr,
+                   cl_uint num_events_in_wait_list,
+                   const cl_event *event_wait_list,
+                   cl_event *event) {
+
+   return EnqueueSVMUnmap(command_queue, svm_ptr, num_events_in_wait_list,
+                          event_wait_list, event, CL_COMMAND_SVM_UNMAP_ARM);
+}
+
+const std::unordered_map<std::string, void *>
+ext_funcs = {
+   // cl_arm_shared_virtual_memory
+   { "clEnqueueSVMFreeARM", reinterpret_cast<void *>(enqueueSVMFreeARM) },
+   { "clEnqueueSVMMapARM", reinterpret_cast<void *>(enqueueSVMMapARM) },
+   { "clEnqueueSVMMemcpyARM", reinterpret_cast<void *>(enqueueSVMMemcpyARM) },
+   { "clEnqueueSVMMemFillARM", reinterpret_cast<void *>(enqueueSVMMemFillARM) },
+   { "clEnqueueSVMUnmapARM", reinterpret_cast<void *>(enqueueSVMUnmapARM) },
+   { "clSetKernelArgSVMPointerARM", reinterpret_cast<void *>(clSetKernelArgSVMPointer) },
+   { "clSetKernelExecInfoARM", reinterpret_cast<void *>(clSetKernelExecInfo) },
+   { "clSVMAllocARM", reinterpret_cast<void *>(clSVMAlloc) },
+   { "clSVMFreeARM", reinterpret_cast<void *>(clSVMFree) },
+
+   // cl_khr_icd
+   { "clIcdGetPlatformIDsKHR", reinterpret_cast<void *>(IcdGetPlatformIDsKHR) },
+};
+
+} // anonymous namespace
+
+void *
+clover::GetExtensionFunctionAddress(const char *p_name) try {
+   return ext_funcs.at(p_name);
+} catch (...) {
+   return nullptr;
 }
 
 cl_int
diff --git a/src/gallium/state_trackers/clover/api/transfer.cpp b/src/gallium/state_trackers/clover/api/transfer.cpp
index d1f540769dd..fa8741e02b4 100644
--- a/src/gallium/state_trackers/clover/api/transfer.cpp
+++ b/src/gallium/state_trackers/clover/api/transfer.cpp
@@ -24,6 +24,7 @@
 
 #include "util/bitscan.h"
 
+#include "api/dispatch.hpp"
 #include "api/util.hpp"
 #include "core/event.hpp"
 #include "core/memory.hpp"
@@ -767,17 +768,19 @@ clEnqueueMigrateMemObjects(cl_command_queue command_queue,
    return CL_INVALID_VALUE;
 }
 
-CLOVER_API cl_int
-clEnqueueSVMFree(cl_command_queue d_q,
-                 cl_uint num_svm_pointers,
-                 void *svm_pointers[],
-                 void (CL_CALLBACK *pfn_free_func) (
-                    cl_command_queue queue, cl_uint num_svm_pointers,
-                    void *svm_pointers[], void *user_data),
-                 void *user_data,
-                 cl_uint num_events_in_wait_list,
-                 const cl_event *event_wait_list,
-                 cl_event *event) try {
+cl_int
+clover::EnqueueSVMFree(cl_command_queue d_q,
+                       cl_uint num_svm_pointers,
+                       void *svm_pointers[],
+                       void (CL_CALLBACK *pfn_free_func) (
+                           cl_command_queue queue, cl_uint num_svm_pointers,
+                           void *svm_pointers[], void *user_data),
+                       void *user_data,
+                       cl_uint num_events_in_wait_list,
+                       const cl_event *event_wait_list,
+                       cl_event *event,
+                       cl_int cmd) try {
+
    if (bool(num_svm_pointers) != bool(svm_pointers))
       return CL_INVALID_VALUE;
 
@@ -801,7 +804,7 @@ clEnqueueSVMFree(cl_command_queue d_q,
       };
    }
 
-   auto hev = create<hard_event>(q, CL_COMMAND_SVM_FREE, deps,
+   auto hev = create<hard_event>(q, cmd, deps,
       [=](clover::event &) mutable {
          pfn_free_func(d_q, num_svm_pointers, svm_pointers_cpy.data(),
                        user_data);
@@ -815,14 +818,32 @@ clEnqueueSVMFree(cl_command_queue d_q,
 }
 
 CLOVER_API cl_int
-clEnqueueSVMMemcpy(cl_command_queue d_q,
-                   cl_bool blocking_copy,
-                   void *dst_ptr,
-                   const void *src_ptr,
-                   size_t size,
-                   cl_uint num_events_in_wait_list,
-                   const cl_event *event_wait_list,
-                   cl_event *event) try {
+clEnqueueSVMFree(cl_command_queue d_q,
+                 cl_uint num_svm_pointers,
+                 void *svm_pointers[],
+                 void (CL_CALLBACK *pfn_free_func) (
+                    cl_command_queue queue, cl_uint num_svm_pointers,
+                    void *svm_pointers[], void *user_data),
+                 void *user_data,
+                 cl_uint num_events_in_wait_list,
+                 const cl_event *event_wait_list,
+                 cl_event *event) {
+
+   return EnqueueSVMFree(d_q, num_svm_pointers, svm_pointers,
+                         pfn_free_func, user_data, num_events_in_wait_list,
+                         event_wait_list, event, CL_COMMAND_SVM_FREE);
+}
+
+cl_int
+clover::EnqueueSVMMemcpy(cl_command_queue d_q,
+                         cl_bool blocking_copy,
+                         void *dst_ptr,
+                         const void *src_ptr,
+                         size_t size,
+                         cl_uint num_events_in_wait_list,
+                         const cl_event *event_wait_list,
+                         cl_event *event,
+                         cl_int cmd) try {
 
    if (dst_ptr == nullptr || src_ptr == nullptr)
       return CL_INVALID_VALUE;
@@ -838,7 +859,7 @@ clEnqueueSVMMemcpy(cl_command_queue d_q,
    validate_common(q, deps);
 
    if (can_emulate) {
-      auto hev = create<hard_event>(q, CL_COMMAND_SVM_MEMCPY, deps,
+      auto hev = create<hard_event>(q, cmd, deps,
          [=](clover::event &) {
             memcpy(dst_ptr, src_ptr, size);
          });
@@ -857,14 +878,31 @@ clEnqueueSVMMemcpy(cl_command_queue d_q,
 }
 
 CLOVER_API cl_int
-clEnqueueSVMMemFill(cl_command_queue d_q,
-                    void *svm_ptr,
-                    const void *pattern,
-                    size_t pattern_size,
-                    size_t size,
-                    cl_uint num_events_in_wait_list,
-                    const cl_event *event_wait_list,
-                    cl_event *event) try {
+clEnqueueSVMMemcpy(cl_command_queue d_q,
+                   cl_bool blocking_copy,
+                   void *dst_ptr,
+                   const void *src_ptr,
+                   size_t size,
+                   cl_uint num_events_in_wait_list,
+                   const cl_event *event_wait_list,
+                   cl_event *event) {
+
+   return EnqueueSVMMemcpy(d_q, blocking_copy, dst_ptr, src_ptr,
+                           size, num_events_in_wait_list, event_wait_list,
+                           event, CL_COMMAND_SVM_MEMCPY);
+}
+
+cl_int
+clover::EnqueueSVMMemFill(cl_command_queue d_q,
+                          void *svm_ptr,
+                          const void *pattern,
+                          size_t pattern_size,
+                          size_t size,
+                          cl_uint num_events_in_wait_list,
+                          const cl_event *event_wait_list,
+                          cl_event *event,
+                          cl_int cmd) try {
+
    if (svm_ptr == nullptr || pattern == nullptr ||
        !util_is_power_of_two_nonzero(pattern_size) ||
        pattern_size > 128 ||
@@ -879,7 +917,7 @@ clEnqueueSVMMemFill(cl_command_queue d_q,
    validate_common(q, deps);
 
    if (can_emulate) {
-      auto hev = create<hard_event>(q, CL_COMMAND_SVM_MEMFILL, deps,
+      auto hev = create<hard_event>(q, cmd, deps,
          [=](clover::event &) {
             void *ptr = svm_ptr;
             for (size_t s = size; s; s -= pattern_size) {
@@ -900,14 +938,30 @@ clEnqueueSVMMemFill(cl_command_queue d_q,
 }
 
 CLOVER_API cl_int
-clEnqueueSVMMap(cl_command_queue d_q,
-                cl_bool blocking_map,
-                cl_map_flags map_flags,
-                void *svm_ptr,
-                size_t size,
-                cl_uint num_events_in_wait_list,
-                const cl_event *event_wait_list,
-                cl_event *event) try {
+clEnqueueSVMMemFill(cl_command_queue d_q,
+                    void *svm_ptr,
+                    const void *pattern,
+                    size_t pattern_size,
+                    size_t size,
+                    cl_uint num_events_in_wait_list,
+                    const cl_event *event_wait_list,
+                    cl_event *event) {
+
+   return EnqueueSVMMemFill(d_q, svm_ptr, pattern, pattern_size,
+                            size, num_events_in_wait_list, event_wait_list,
+                            event, CL_COMMAND_SVM_MEMFILL);
+}
+
+cl_int
+clover::EnqueueSVMMap(cl_command_queue d_q,
+                      cl_bool blocking_map,
+                      cl_map_flags map_flags,
+                      void *svm_ptr,
+                      size_t size,
+                      cl_uint num_events_in_wait_list,
+                      const cl_event *event_wait_list,
+                      cl_event *event,
+                      cl_int cmd) try {
 
    if (svm_ptr == nullptr || size == 0)
       return CL_INVALID_VALUE;
@@ -919,7 +973,7 @@ clEnqueueSVMMap(cl_command_queue d_q,
    validate_common(q, deps);
 
    if (can_emulate) {
-      auto hev = create<hard_event>(q, CL_COMMAND_SVM_MAP, deps,
+      auto hev = create<hard_event>(q, cmd, deps,
          [](clover::event &) { });
 
       ret_object(event, hev);
@@ -934,11 +988,27 @@ clEnqueueSVMMap(cl_command_queue d_q,
 }
 
 CLOVER_API cl_int
-clEnqueueSVMUnmap(cl_command_queue d_q,
-                  void *svm_ptr,
-                  cl_uint num_events_in_wait_list,
-                  const cl_event *event_wait_list,
-                  cl_event *event) try {
+clEnqueueSVMMap(cl_command_queue d_q,
+                cl_bool blocking_map,
+                cl_map_flags map_flags,
+                void *svm_ptr,
+                size_t size,
+                cl_uint num_events_in_wait_list,
+                const cl_event *event_wait_list,
+                cl_event *event) {
+
+   return EnqueueSVMMap(d_q, blocking_map, map_flags, svm_ptr, size,
+                        num_events_in_wait_list, event_wait_list, event,
+                        CL_COMMAND_SVM_MAP);
+}
+
+cl_int
+clover::EnqueueSVMUnmap(cl_command_queue d_q,
+                        void *svm_ptr,
+                        cl_uint num_events_in_wait_list,
+                        const cl_event *event_wait_list,
+                        cl_event *event,
+                        cl_int cmd) try {
 
    if (svm_ptr == nullptr)
       return CL_INVALID_VALUE;
@@ -950,7 +1020,7 @@ clEnqueueSVMUnmap(cl_command_queue d_q,
    validate_common(q, deps);
 
    if (can_emulate) {
-      auto hev = create<hard_event>(q, CL_COMMAND_SVM_UNMAP, deps,
+      auto hev = create<hard_event>(q, cmd, deps,
          [](clover::event &) { });
 
       ret_object(event, hev);
@@ -965,6 +1035,17 @@ clEnqueueSVMUnmap(cl_command_queue d_q,
 }
 
 CLOVER_API cl_int
+clEnqueueSVMUnmap(cl_command_queue d_q,
+                  void *svm_ptr,
+                  cl_uint num_events_in_wait_list,
+                  const cl_event *event_wait_list,
+                  cl_event *event) {
+
+   return EnqueueSVMUnmap(d_q, svm_ptr, num_events_in_wait_list,
+                          event_wait_list, event, CL_COMMAND_SVM_UNMAP);
+}
+
+CLOVER_API cl_int
 clEnqueueSVMMigrateMem(cl_command_queue d_q,
                        cl_uint num_svm_pointers,
                        const void **svm_pointers,