/* * Copyright © 2014 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. * * Authors: * Connor Abbott (cwabbott0@gmail.com) * */ #include "nir.h" #include "nir_builder.h" static nir_ssa_def* build_local_group_size(nir_builder *b, unsigned bit_size) { nir_ssa_def *local_size; /* * If the local work group size is variable it can't be lowered at this * point, but its intrinsic can still be used. */ if (b->shader->info.cs.local_size_variable) { local_size = nir_load_local_group_size(b); } else { /* using a 32 bit constant is safe here as no device/driver needs more * than 32 bits for the local size */ nir_const_value local_size_const; memset(&local_size_const, 0, sizeof(local_size_const)); local_size_const.u32[0] = b->shader->info.cs.local_size[0]; local_size_const.u32[1] = b->shader->info.cs.local_size[1]; local_size_const.u32[2] = b->shader->info.cs.local_size[2]; local_size = nir_build_imm(b, 3, 32, local_size_const); } return nir_u2u(b, local_size, bit_size); } static nir_ssa_def * build_local_invocation_id(nir_builder *b, unsigned bit_size) { /* If lower_cs_local_id_from_index is true, then we derive the local * index from the local id. */ if (b->shader->options->lower_cs_local_id_from_index) { /* We lower gl_LocalInvocationID from gl_LocalInvocationIndex based * on this formula: * * gl_LocalInvocationID.x = * gl_LocalInvocationIndex % gl_WorkGroupSize.x; * gl_LocalInvocationID.y = * (gl_LocalInvocationIndex / gl_WorkGroupSize.x) % * gl_WorkGroupSize.y; * gl_LocalInvocationID.z = * (gl_LocalInvocationIndex / * (gl_WorkGroupSize.x * gl_WorkGroupSize.y)) % * gl_WorkGroupSize.z; * * However, the final % gl_WorkGroupSize.z does nothing unless we * accidentally end up with a gl_LocalInvocationIndex that is too * large so it can safely be omitted. */ nir_ssa_def *local_index = nir_load_local_invocation_index(b); nir_ssa_def *local_size = build_local_group_size(b, 32); /* Because no hardware supports a local workgroup size greater than * about 1K, this calculation can be done in 32-bit and can save some * 64-bit arithmetic. */ nir_ssa_def *id_x, *id_y, *id_z; id_x = nir_umod(b, local_index, nir_channel(b, local_size, 0)); id_y = nir_umod(b, nir_udiv(b, local_index, nir_channel(b, local_size, 0)), nir_channel(b, local_size, 1)); id_z = nir_udiv(b, local_index, nir_imul(b, nir_channel(b, local_size, 0), nir_channel(b, local_size, 1))); return nir_u2u(b, nir_vec3(b, id_x, id_y, id_z), bit_size); } else { return nir_u2u(b, nir_load_local_invocation_id(b), bit_size); } } static nir_ssa_def* build_global_group_size(nir_builder *b, unsigned bit_size) { nir_ssa_def *group_size = build_local_group_size(b, bit_size); nir_ssa_def *num_work_groups = nir_u2u(b, nir_load_num_work_groups(b), bit_size); return nir_imul(b, group_size, num_work_groups); } static nir_ssa_def* build_global_invocation_id(nir_builder *b, unsigned bit_size) { /* From the GLSL man page for gl_GlobalInvocationID: * * "The value of gl_GlobalInvocationID is equal to * gl_WorkGroupID * gl_WorkGroupSize + gl_LocalInvocationID" */ nir_ssa_def *group_size = build_local_group_size(b, bit_size); nir_ssa_def *group_id = nir_u2u(b, nir_load_work_group_id(b), bit_size); nir_ssa_def *local_id = build_local_invocation_id(b, bit_size); return nir_iadd(b, nir_imul(b, group_id, group_size), local_id); } static bool convert_block(nir_block *block, nir_builder *b) { bool progress = false; nir_foreach_instr_safe(instr, block) { if (instr->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *load_deref = nir_instr_as_intrinsic(instr); if (load_deref->intrinsic != nir_intrinsic_load_deref) continue; nir_deref_instr *deref = nir_src_as_deref(load_deref->src[0]); if (deref->mode != nir_var_system_value) continue; if (deref->deref_type != nir_deref_type_var) { /* The only one system value that is an array and that is * gl_SampleMask which is always an array of one element. */ assert(deref->deref_type == nir_deref_type_array); deref = nir_deref_instr_parent(deref); assert(deref->deref_type == nir_deref_type_var); assert(deref->var->data.location == SYSTEM_VALUE_SAMPLE_MASK_IN); } nir_variable *var = deref->var; b->cursor = nir_after_instr(&load_deref->instr); unsigned bit_size = nir_dest_bit_size(load_deref->dest); nir_ssa_def *sysval = NULL; switch (var->data.location) { case SYSTEM_VALUE_GLOBAL_INVOCATION_ID: { sysval = build_global_invocation_id(b, bit_size); break; } case SYSTEM_VALUE_GLOBAL_INVOCATION_INDEX: { nir_ssa_def *global_id = build_global_invocation_id(b, bit_size); nir_ssa_def *global_size = build_global_group_size(b, bit_size); /* index = id.x + ((id.y + (id.z * size.y)) * size.x) */ sysval = nir_imul(b, nir_channel(b, global_id, 2), nir_channel(b, global_size, 1)); sysval = nir_iadd(b, nir_channel(b, global_id, 1), sysval); sysval = nir_imul(b, nir_channel(b, global_size, 0), sysval); sysval = nir_iadd(b, nir_channel(b, global_id, 0), sysval); break; } case SYSTEM_VALUE_LOCAL_INVOCATION_INDEX: { /* If lower_cs_local_index_from_id is true, then we derive the local * index from the local id. */ if (!b->shader->options->lower_cs_local_index_from_id) break; /* From the GLSL man page for gl_LocalInvocationIndex: * * "The value of gl_LocalInvocationIndex is equal to * gl_LocalInvocationID.z * gl_WorkGroupSize.x * * gl_WorkGroupSize.y + gl_LocalInvocationID.y * * gl_WorkGroupSize.x + gl_LocalInvocationID.x" */ nir_ssa_def *local_id = nir_load_local_invocation_id(b); nir_ssa_def *size_x = nir_imm_int(b, b->shader->info.cs.local_size[0]); nir_ssa_def *size_y = nir_imm_int(b, b->shader->info.cs.local_size[1]); /* Because no hardware supports a local workgroup size greater than * about 1K, this calculation can be done in 32-bit and can save some * 64-bit arithmetic. */ sysval = nir_imul(b, nir_channel(b, local_id, 2), nir_imul(b, size_x, size_y)); sysval = nir_iadd(b, sysval, nir_imul(b, nir_channel(b, local_id, 1), size_x)); sysval = nir_iadd(b, sysval, nir_channel(b, local_id, 0)); sysval = nir_u2u(b, sysval, bit_size); break; } case SYSTEM_VALUE_LOCAL_INVOCATION_ID: sysval = build_local_invocation_id(b, bit_size); break; case SYSTEM_VALUE_LOCAL_GROUP_SIZE: { sysval = build_local_group_size(b, bit_size); break; } case SYSTEM_VALUE_VERTEX_ID: if (b->shader->options->vertex_id_zero_based) { sysval = nir_iadd(b, nir_load_vertex_id_zero_base(b), nir_load_first_vertex(b)); } else { sysval = nir_load_vertex_id(b); } break; case SYSTEM_VALUE_BASE_VERTEX: /** * From the OpenGL 4.6 (11.1.3.9 Shader Inputs) specification: * * "gl_BaseVertex holds the integer value passed to the baseVertex * parameter to the command that resulted in the current shader * invocation. In the case where the command has no baseVertex * parameter, the value of gl_BaseVertex is zero." */ if (b->shader->options->lower_base_vertex) sysval = nir_iand(b, nir_load_is_indexed_draw(b), nir_load_first_vertex(b)); break; case SYSTEM_VALUE_HELPER_INVOCATION: if (b->shader->options->lower_helper_invocation) { nir_ssa_def *tmp; tmp = nir_ishl(b, nir_imm_int(b, 1), nir_load_sample_id_no_per_sample(b)); tmp = nir_iand(b, nir_load_sample_mask_in(b), tmp); sysval = nir_inot(b, nir_i2b(b, tmp)); } break; case SYSTEM_VALUE_INSTANCE_INDEX: sysval = nir_iadd(b, nir_load_instance_id(b), nir_load_base_instance(b)); break; case SYSTEM_VALUE_SUBGROUP_EQ_MASK: case SYSTEM_VALUE_SUBGROUP_GE_MASK: case SYSTEM_VALUE_SUBGROUP_GT_MASK: case SYSTEM_VALUE_SUBGROUP_LE_MASK: case SYSTEM_VALUE_SUBGROUP_LT_MASK: { nir_intrinsic_op op = nir_intrinsic_from_system_value(var->data.location); nir_intrinsic_instr *load = nir_intrinsic_instr_create(b->shader, op); nir_ssa_dest_init_for_type(&load->instr, &load->dest, var->type, NULL); load->num_components = load->dest.ssa.num_components; nir_builder_instr_insert(b, &load->instr); sysval = &load->dest.ssa; break; } case SYSTEM_VALUE_DEVICE_INDEX: if (b->shader->options->lower_device_index_to_zero) sysval = nir_imm_int(b, 0); break; case SYSTEM_VALUE_GLOBAL_GROUP_SIZE: { sysval = build_global_group_size(b, bit_size); break; } default: break; } if (sysval == NULL) { nir_intrinsic_op sysval_op = nir_intrinsic_from_system_value(var->data.location); sysval = nir_load_system_value(b, sysval_op, 0, load_deref->dest.ssa.bit_size); } nir_ssa_def_rewrite_uses(&load_deref->dest.ssa, nir_src_for_ssa(sysval)); nir_instr_remove(&load_deref->instr); progress = true; } return progress; } static bool convert_impl(nir_function_impl *impl) { bool progress = false; nir_builder builder; nir_builder_init(&builder, impl); nir_foreach_block(block, impl) { progress |= convert_block(block, &builder); } nir_metadata_preserve(impl, nir_metadata_block_index | nir_metadata_dominance); return progress; } bool nir_lower_system_values(nir_shader *shader) { bool progress = false; nir_foreach_function(function, shader) { if (function->impl) progress = convert_impl(function->impl) || progress; } /* We're going to delete the variables so we need to clean up all those * derefs we left lying around. */ nir_remove_dead_derefs(shader); exec_list_make_empty(&shader->system_values); return progress; }