clover: add support for passing kernels as nir to the driver

v2: minor formatting fixes
v3: call glsl_type_singleton_init_or_ref and glsl_type_singleton_decref
v4: capitalize and punctuate comments
    fix text_executable -> text_intermediate in TODO
    make glsl_type_singleton wrapper static
v5: rewrite how we run the nir passes
v6: fix unhandled case switch warning in st/mesa

Signed-off-by: Karol Herbst <[email protected]>
Reviewed-by: Francisco Jerez <[email protected]> (v4)

7 files changed, 246 insertions, 7 deletions

diff --git a/src/gallium/include/pipe/p_defines.h b/src/gallium/include/pipe/p_defines.h
index de0d88f6548..0d4adcf6f4b 100644
--- a/src/gallium/include/pipe/p_defines.h
+++ b/src/gallium/include/pipe/p_defines.h
@@ -1004,6 +1004,7 @@ enum pipe_shader_ir
    PIPE_SHADER_IR_TGSI = 0,
    PIPE_SHADER_IR_NATIVE,
    PIPE_SHADER_IR_NIR,
+   PIPE_SHADER_IR_NIR_SERIALIZED,
 };
 
 /**
diff --git a/src/gallium/state_trackers/clover/core/compiler.hpp b/src/gallium/state_trackers/clover/core/compiler.hpp
index 4ed20061627..6ef30df9b7f 100644
--- a/src/gallium/state_trackers/clover/core/compiler.hpp
+++ b/src/gallium/state_trackers/clover/core/compiler.hpp
@@ -26,6 +26,8 @@
 #include "core/device.hpp"
 #include "core/module.hpp"
 #include "llvm/invocation.hpp"
+#include "nir/invocation.hpp"
+#include "spirv/invocation.hpp"
 
 namespace clover {
    namespace compiler {
@@ -34,6 +36,10 @@ namespace clover {
                       const device &dev, const std::string &opts,
                       std::string &log) {
          switch (dev.ir_format()) {
+#ifdef HAVE_CLOVER_SPIRV
+         case PIPE_SHADER_IR_NIR_SERIALIZED:
+            return llvm::compile_to_spirv(source, headers, dev, opts, log);
+#endif
          case PIPE_SHADER_IR_NATIVE:
             return llvm::compile_program(source, headers, dev, opts, log);
          default:
@@ -46,6 +52,9 @@ namespace clover {
       link_program(const std::vector<module> &ms, const device &dev,
                    const std::string &opts, std::string &log) {
          switch (dev.ir_format()) {
+         case PIPE_SHADER_IR_NIR_SERIALIZED:
+            return nir::spirv_to_nir(spirv::link_program(ms, dev, opts, log),
+                                     dev, log);
          case PIPE_SHADER_IR_NATIVE:
             return llvm::link_program(ms, dev, opts, log);
          default:
diff --git a/src/gallium/state_trackers/clover/core/device.cpp b/src/gallium/state_trackers/clover/core/device.cpp
index c7aea827938..298cde278b4 100644
--- a/src/gallium/state_trackers/clover/core/device.cpp
+++ b/src/gallium/state_trackers/clover/core/device.cpp
@@ -46,12 +46,17 @@ namespace {
 device::device(clover::platform &platform, pipe_loader_device *ldev) :
    platform(platform), ldev(ldev) {
    pipe = pipe_loader_create_screen(ldev);
-   if (!pipe || !pipe->get_param(pipe, PIPE_CAP_COMPUTE) ||
-       !supports_ir(PIPE_SHADER_IR_NATIVE)) {
-      if (pipe)
-         pipe->destroy(pipe);
-      throw error(CL_INVALID_DEVICE);
+   if (pipe && pipe->get_param(pipe, PIPE_CAP_COMPUTE)) {
+      if (supports_ir(PIPE_SHADER_IR_NATIVE))
+         return;
+#ifdef HAVE_CLOVER_SPIRV
+      if (supports_ir(PIPE_SHADER_IR_NIR_SERIALIZED))
+         return;
+#endif
    }
+   if (pipe)
+      pipe->destroy(pipe);
+   throw error(CL_INVALID_DEVICE);
 }
 
 device::~device() {
@@ -246,7 +251,11 @@ device::vendor_name() const {
 
 enum pipe_shader_ir
 device::ir_format() const {
-   return PIPE_SHADER_IR_NATIVE;
+   if (supports_ir(PIPE_SHADER_IR_NATIVE))
+      return PIPE_SHADER_IR_NATIVE;
+
+   assert(supports_ir(PIPE_SHADER_IR_NIR_SERIALIZED));
+   return PIPE_SHADER_IR_NIR_SERIALIZED;
 }
 
 std::string
@@ -294,3 +303,8 @@ device::supported_extensions() const {
       + std::string(has_doubles() ? " cl_khr_fp64" : "")
       + std::string(has_halves() ? " cl_khr_fp16" : "");
 }
+
+const void *
+device::get_compiler_options(enum pipe_shader_ir ir) const {
+   return pipe->get_compiler_options(pipe, ir, PIPE_SHADER_COMPUTE);
+}
diff --git a/src/gallium/state_trackers/clover/core/device.hpp b/src/gallium/state_trackers/clover/core/device.hpp
index a7084e863f4..828dfaa65e6 100644
--- a/src/gallium/state_trackers/clover/core/device.hpp
+++ b/src/gallium/state_trackers/clover/core/device.hpp
@@ -90,6 +90,7 @@ namespace clover {
       friend class hard_event;
       friend std::set<cl_image_format>
       supported_formats(const context &, cl_mem_object_type);
+      const void *get_compiler_options(enum pipe_shader_ir ir) const;
 
       clover::platform &platform;
 
diff --git a/src/gallium/state_trackers/clover/meson.build b/src/gallium/state_trackers/clover/meson.build
index 13d554c3fbf..04c6f1ea0eb 100644
--- a/src/gallium/state_trackers/clover/meson.build
+++ b/src/gallium/state_trackers/clover/meson.build
@@ -66,6 +66,15 @@ libclspirv = static_library(
   override_options : clover_cpp_std,
 )
 
+libclnir = static_library(
+  'clnir',
+  [files('nir/invocation.cpp', 'nir/invocation.hpp'), nir_opcodes_h],
+  include_directories : [clover_incs, inc_mesa],
+  cpp_args : [clover_spirv_cpp_args, cpp_vis_args],
+  link_with : [libnir],
+  override_options : clover_cpp_std,
+)
+
 clover_files = files(
   'api/context.cpp',
   'api/device.cpp',
@@ -127,6 +136,6 @@ libclover = static_library(
   [clover_files, sha1_h],
   include_directories : clover_incs,
   cpp_args : [clover_spirv_cpp_args, clover_cpp_args, cpp_vis_args],
-  link_with : [libclllvm, libclspirv],
+  link_with : [libclllvm, libclspirv, libclnir],
   override_options : clover_cpp_std,
 )
diff --git a/src/gallium/state_trackers/clover/nir/invocation.cpp b/src/gallium/state_trackers/clover/nir/invocation.cpp
new file mode 100644
index 00000000000..0ec1c9a3c01
--- /dev/null
+++ b/src/gallium/state_trackers/clover/nir/invocation.cpp
@@ -0,0 +1,169 @@
+//
+// Copyright 2019 Karol Herbst
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the "Software"),
+// to deal in the Software without restriction, including without limitation
+// the rights to use, copy, modify, merge, publish, distribute, sublicense,
+// and/or sell copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+//
+
+#include "invocation.hpp"
+
+#include <tuple>
+
+#include "core/device.hpp"
+#include "core/error.hpp"
+#include "pipe/p_state.h"
+#include "util/algorithm.hpp"
+#include "util/functional.hpp"
+
+#include <compiler/glsl_types.h>
+#include <compiler/nir/nir_serialize.h>
+#include <compiler/spirv/nir_spirv.h>
+#include <util/u_math.h>
+
+using namespace clover;
+
+#ifdef HAVE_CLOVER_SPIRV
+
+// Refs and unrefs the glsl_type_singleton.
+static class glsl_type_ref {
+public:
+   glsl_type_ref() {
+      glsl_type_singleton_init_or_ref();
+   }
+
+   ~glsl_type_ref() {
+      glsl_type_singleton_decref();
+   }
+} glsl_type_ref;
+
+static const nir_shader_compiler_options *
+dev_get_nir_compiler_options(const device &dev)
+{
+   const void *co = dev.get_compiler_options(PIPE_SHADER_IR_NIR);
+   return static_cast<const nir_shader_compiler_options*>(co);
+}
+
+module clover::nir::spirv_to_nir(const module &mod, const device &dev,
+                                 std::string &r_log)
+{
+   const struct spirv_to_nir_options spirv_options = {
+      .caps = {
+         .address = true,
+         .float64 = true,
+         .int8 = true,
+         .int16 = true,
+         .int64 = true,
+         .kernel = true,
+      },
+   };
+
+   module m;
+   // We only insert one section.
+   assert(mod.secs.size() == 1);
+   auto &section = mod.secs[0];
+
+   module::resource_id section_id = 0;
+   for (const auto &sym : mod.syms) {
+      assert(sym.section == 0);
+
+      const auto *binary =
+         reinterpret_cast<const pipe_binary_program_header *>(section.data.data());
+      const uint32_t *data = reinterpret_cast<const uint32_t *>(binary->blob);
+      const size_t num_words = binary->num_bytes / 4;
+      const char *name = sym.name.c_str();
+      auto *compiler_options = dev_get_nir_compiler_options(dev);
+
+      nir_shader *nir = spirv_to_nir(data, num_words, nullptr, 0,
+                                     MESA_SHADER_KERNEL, name,
+                                     &spirv_options, compiler_options);
+
+      nir->info.cs.local_size_variable = true;
+      nir_validate_shader(nir, "clover");
+
+      // Calculate input offsets.
+      unsigned offset = 0;
+      nir_foreach_variable_safe(var, &nir->inputs) {
+         offset = align(offset, glsl_get_cl_alignment(var->type));
+         var->data.driver_location = offset;
+         offset += glsl_get_cl_size(var->type);
+      }
+
+      // Inline all functions first.
+      // according to the comment on nir_inline_functions
+      NIR_PASS_V(nir, nir_lower_constant_initializers, nir_var_function_temp);
+      NIR_PASS_V(nir, nir_lower_returns);
+      NIR_PASS_V(nir, nir_inline_functions);
+      NIR_PASS_V(nir, nir_opt_deref);
+
+      // Pick off the single entrypoint that we want.
+      foreach_list_typed_safe(nir_function, func, node, &nir->functions) {
+         if (!func->is_entrypoint)
+            exec_node_remove(&func->node);
+      }
+      assert(exec_list_length(&nir->functions) == 1);
+
+      nir_validate_shader(nir, "clover after function inlining");
+
+      NIR_PASS_V(nir, nir_lower_constant_initializers,
+                 static_cast<nir_variable_mode>(~nir_var_function_temp));
+
+      // copy propagate to prepare for lower_explicit_io
+      NIR_PASS_V(nir, nir_split_var_copies);
+      NIR_PASS_V(nir, nir_opt_copy_prop_vars);
+      NIR_PASS_V(nir, nir_lower_var_copies);
+      NIR_PASS_V(nir, nir_lower_vars_to_ssa);
+      NIR_PASS_V(nir, nir_opt_dce);
+
+      nir_variable_mode modes = (nir_variable_mode)(
+         nir_var_shader_in |
+         nir_var_mem_global |
+         nir_var_mem_shared);
+      nir_address_format format = nir->info.cs.ptr_size == 64 ?
+         nir_address_format_64bit_global : nir_address_format_32bit_global;
+      NIR_PASS_V(nir, nir_lower_explicit_io, modes, format);
+
+      NIR_PASS_V(nir, nir_lower_system_values);
+      if (compiler_options->lower_int64_options)
+         NIR_PASS_V(nir, nir_lower_int64,
+                    compiler_options->lower_int64_options);
+
+      NIR_PASS_V(nir, nir_opt_dce);
+
+      struct blob blob;
+      blob_init(&blob);
+      nir_serialize(&blob, nir);
+
+      const pipe_binary_program_header header { uint32_t(blob.size) };
+      module::section text { section_id, module::section::text_executable, header.num_bytes, {} };
+      text.data.insert(text.data.end(), reinterpret_cast<const char *>(&header),
+                       reinterpret_cast<const char *>(&header) + sizeof(header));
+      text.data.insert(text.data.end(), blob.data, blob.data + blob.size);
+
+      m.syms.emplace_back(sym.name, section_id, 0, sym.args);
+      m.secs.push_back(text);
+      section_id++;
+   }
+   return m;
+}
+#else
+module clover::nir::spirv_to_nir(const module &mod, const device &dev, std::string &r_log)
+{
+   r_log += "SPIR-V support in clover is not enabled.\n";
+   throw error(CL_LINKER_NOT_AVAILABLE);
+}
+#endif
diff --git a/src/gallium/state_trackers/clover/nir/invocation.hpp b/src/gallium/state_trackers/clover/nir/invocation.hpp
new file mode 100644
index 00000000000..41407a79765
--- /dev/null
+++ b/src/gallium/state_trackers/clover/nir/invocation.hpp
@@ -0,0 +1,36 @@
+//
+// Copyright 2019 Karol Herbst
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the "Software"),
+// to deal in the Software without restriction, including without limitation
+// the rights to use, copy, modify, merge, publish, distribute, sublicense,
+// and/or sell copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+//
+
+#ifndef CLOVER_NIR_INVOCATION_HPP
+#define CLOVER_NIR_INVOCATION_HPP
+
+#include "core/module.hpp"
+
+namespace clover {
+   class device;
+   namespace nir {
+      // converts a given spirv module to nir
+      module spirv_to_nir(const module &mod, const device &dev, std::string &r_log);
+   }
+}
+
+#endif