summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/amd/common/ac_nir_to_llvm.c120
1 files changed, 41 insertions, 79 deletions
diff --git a/src/amd/common/ac_nir_to_llvm.c b/src/amd/common/ac_nir_to_llvm.c
index ae21be4d8ff..2a5f916ccc9 100644
--- a/src/amd/common/ac_nir_to_llvm.c
+++ b/src/amd/common/ac_nir_to_llvm.c
@@ -142,10 +142,6 @@ struct ac_tex_info {
bool has_offset;
};
-static LLVMValueRef
-emit_llvm_intrinsic(struct nir_to_llvm_context *ctx, const char *name,
- LLVMTypeRef return_type, LLVMValueRef *params,
- unsigned param_count, unsigned attr_mask);
static LLVMValueRef get_sampler_desc(struct nir_to_llvm_context *ctx,
nir_deref_var *deref,
enum desc_type desc_type);
@@ -797,7 +793,7 @@ static LLVMValueRef emit_intrin_1f_param(struct nir_to_llvm_context *ctx,
LLVMValueRef params[] = {
to_float(ctx, src0),
};
- return emit_llvm_intrinsic(ctx, intrin, ctx->f32, params, 1, AC_FUNC_ATTR_READNONE);
+ return ac_emit_llvm_intrinsic(&ctx->ac, intrin, ctx->f32, params, 1, AC_FUNC_ATTR_READNONE);
}
static LLVMValueRef emit_intrin_2f_param(struct nir_to_llvm_context *ctx,
@@ -808,7 +804,7 @@ static LLVMValueRef emit_intrin_2f_param(struct nir_to_llvm_context *ctx,
to_float(ctx, src0),
to_float(ctx, src1),
};
- return emit_llvm_intrinsic(ctx, intrin, ctx->f32, params, 2, AC_FUNC_ATTR_READNONE);
+ return ac_emit_llvm_intrinsic(&ctx->ac, intrin, ctx->f32, params, 2, AC_FUNC_ATTR_READNONE);
}
static LLVMValueRef emit_intrin_3f_param(struct nir_to_llvm_context *ctx,
@@ -820,7 +816,7 @@ static LLVMValueRef emit_intrin_3f_param(struct nir_to_llvm_context *ctx,
to_float(ctx, src1),
to_float(ctx, src2),
};
- return emit_llvm_intrinsic(ctx, intrin, ctx->f32, params, 3, AC_FUNC_ATTR_READNONE);
+ return ac_emit_llvm_intrinsic(&ctx->ac, intrin, ctx->f32, params, 3, AC_FUNC_ATTR_READNONE);
}
static LLVMValueRef emit_bcsel(struct nir_to_llvm_context *ctx,
@@ -846,13 +842,13 @@ static LLVMValueRef emit_find_lsb(struct nir_to_llvm_context *ctx,
*/
LLVMConstInt(ctx->i32, 1, false),
};
- return emit_llvm_intrinsic(ctx, "llvm.cttz.i32", ctx->i32, params, 2, AC_FUNC_ATTR_READNONE);
+ return ac_emit_llvm_intrinsic(&ctx->ac, "llvm.cttz.i32", ctx->i32, params, 2, AC_FUNC_ATTR_READNONE);
}
static LLVMValueRef emit_ifind_msb(struct nir_to_llvm_context *ctx,
LLVMValueRef src0)
{
- LLVMValueRef msb = emit_llvm_intrinsic(ctx, "llvm.AMDGPU.flbit.i32",
+ LLVMValueRef msb = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.AMDGPU.flbit.i32",
ctx->i32, &src0, 1,
AC_FUNC_ATTR_READNONE);
@@ -878,7 +874,7 @@ static LLVMValueRef emit_ufind_msb(struct nir_to_llvm_context *ctx,
src0,
ctx->i32one,
};
- LLVMValueRef msb = emit_llvm_intrinsic(ctx, "llvm.ctlz.i32",
+ LLVMValueRef msb = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.ctlz.i32",
ctx->i32, args, ARRAY_SIZE(args),
AC_FUNC_ATTR_READNONE);
@@ -942,7 +938,7 @@ static LLVMValueRef emit_ffract(struct nir_to_llvm_context *ctx,
LLVMValueRef params[] = {
fsrc0,
};
- LLVMValueRef floor = emit_llvm_intrinsic(ctx, intr,
+ LLVMValueRef floor = ac_emit_llvm_intrinsic(&ctx->ac, intr,
ctx->f32, params, 1,
AC_FUNC_ATTR_READNONE);
return LLVMBuildFSub(ctx->builder, fsrc0, floor, "");
@@ -959,7 +955,7 @@ static LLVMValueRef emit_uint_carry(struct nir_to_llvm_context *ctx,
ret_type = LLVMStructTypeInContext(ctx->context, types,
2, true);
- res = emit_llvm_intrinsic(ctx, intrin, ret_type,
+ res = ac_emit_llvm_intrinsic(&ctx->ac, intrin, ret_type,
params, 2, AC_FUNC_ATTR_READNONE);
res = LLVMBuildExtractValue(ctx->builder, res, 1, "");
@@ -1005,7 +1001,7 @@ static LLVMValueRef emit_bitfield_extract(struct nir_to_llvm_context *ctx,
{
LLVMValueRef result;
LLVMValueRef icond = LLVMBuildICmp(ctx->builder, LLVMIntEQ, srcs[2], LLVMConstInt(ctx->i32, 32, false), "");
- result = emit_llvm_intrinsic(ctx, intrin, ctx->i32, srcs, 3, AC_FUNC_ATTR_READNONE);
+ result = ac_emit_llvm_intrinsic(&ctx->ac, intrin, ctx->i32, srcs, 3, AC_FUNC_ATTR_READNONE);
result = LLVMBuildSelect(ctx->builder, icond, srcs[0], result, "");
return result;
@@ -1110,11 +1106,11 @@ static LLVMValueRef get_thread_id(struct nir_to_llvm_context *ctx)
LLVMValueRef tid_args[2];
tid_args[0] = LLVMConstInt(ctx->i32, 0xffffffff, false);
tid_args[1] = ctx->i32zero;
- tid_args[1] = emit_llvm_intrinsic(ctx,
+ tid_args[1] = ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.amdgcn.mbcnt.lo", ctx->i32,
tid_args, 2, AC_FUNC_ATTR_READNONE);
- tid = emit_llvm_intrinsic(ctx,
+ tid = ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.amdgcn.mbcnt.hi", ctx->i32,
tid_args, 2, AC_FUNC_ATTR_READNONE);
set_range_metadata(ctx, tid, 0, 64);
@@ -1191,13 +1187,13 @@ static LLVMValueRef emit_ddxy(struct nir_to_llvm_context *ctx,
args[0] = LLVMBuildMul(ctx->builder, tl_tid,
LLVMConstInt(ctx->i32, 4, false), "");
args[1] = src0;
- tl = emit_llvm_intrinsic(ctx, "llvm.amdgcn.ds.bpermute",
+ tl = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.ds.bpermute",
ctx->i32, args, 2,
AC_FUNC_ATTR_READNONE);
args[0] = LLVMBuildMul(ctx->builder, trbl_tid,
LLVMConstInt(ctx->i32, 4, false), "");
- trbl = emit_llvm_intrinsic(ctx, "llvm.amdgcn.ds.bpermute",
+ trbl = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.ds.bpermute",
ctx->i32, args, 2,
AC_FUNC_ATTR_READNONE);
} else {
@@ -1477,10 +1473,10 @@ static void visit_alu(struct nir_to_llvm_context *ctx, nir_alu_instr *instr)
result = emit_bitfield_insert(ctx, src[0], src[1], src[2], src[3]);
break;
case nir_op_bitfield_reverse:
- result = emit_llvm_intrinsic(ctx, "llvm.bitreverse.i32", ctx->i32, src, 1, AC_FUNC_ATTR_READNONE);
+ result = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.bitreverse.i32", ctx->i32, src, 1, AC_FUNC_ATTR_READNONE);
break;
case nir_op_bit_count:
- result = emit_llvm_intrinsic(ctx, "llvm.ctpop.i32", ctx->i32, src, 1, AC_FUNC_ATTR_READNONE);
+ result = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.ctpop.i32", ctx->i32, src, 1, AC_FUNC_ATTR_READNONE);
break;
case nir_op_vec2:
case nir_op_vec3:
@@ -1606,40 +1602,6 @@ static LLVMValueRef cast_ptr(struct nir_to_llvm_context *ctx, LLVMValueRef ptr,
}
static LLVMValueRef
-emit_llvm_intrinsic(struct nir_to_llvm_context *ctx, const char *name,
- LLVMTypeRef return_type, LLVMValueRef *params,
- unsigned param_count, unsigned attrib_mask)
-{
- LLVMValueRef function;
-
- function = LLVMGetNamedFunction(ctx->module, name);
- if (!function) {
- LLVMTypeRef param_types[32], function_type;
- unsigned i;
-
- assert(param_count <= 32);
-
- for (i = 0; i < param_count; ++i) {
- assert(params[i]);
- param_types[i] = LLVMTypeOf(params[i]);
- }
- function_type =
- LLVMFunctionType(return_type, param_types, param_count, 0);
- function = LLVMAddFunction(ctx->module, name, function_type);
-
- LLVMSetFunctionCallConv(function, LLVMCCallConv);
- LLVMSetLinkage(function, LLVMExternalLinkage);
-
- attrib_mask |= AC_FUNC_ATTR_NOUNWIND;
- while (attrib_mask) {
- enum ac_func_attr attr = 1u << u_bit_scan(&attrib_mask);
- ac_add_function_attr(function, -1, attr);
- }
- }
- return LLVMBuildCall(ctx->builder, function, params, param_count, "");
-}
-
-static LLVMValueRef
get_buffer_size(struct nir_to_llvm_context *ctx, LLVMValueRef descriptor, bool in_elements)
{
LLVMValueRef size =
@@ -1708,7 +1670,7 @@ static LLVMValueRef radv_lower_gather4_integer(struct nir_to_llvm_context *ctx,
txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /* slc */
txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /* tfe */
txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /* lwe */
- size = emit_llvm_intrinsic(ctx, "llvm.SI.getresinfo.i32", ctx->v4i32,
+ size = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.getresinfo.i32", ctx->v4i32,
txq_args, txq_arg_count,
AC_FUNC_ATTR_READNONE);
@@ -1733,7 +1695,7 @@ static LLVMValueRef radv_lower_gather4_integer(struct nir_to_llvm_context *ctx,
}
tinfo->args[0] = coord;
- return emit_llvm_intrinsic(ctx, intr_name, tinfo->dst_type, tinfo->args, tinfo->arg_count,
+ return ac_emit_llvm_intrinsic(&ctx->ac, intr_name, tinfo->dst_type, tinfo->args, tinfo->arg_count,
AC_FUNC_ATTR_READNONE | AC_FUNC_ATTR_NOUNWIND);
}
@@ -1801,7 +1763,7 @@ static LLVMValueRef build_tex_intrinsic(struct nir_to_llvm_context *ctx,
(int)has_offset + (int)is_shadow);
}
}
- return emit_llvm_intrinsic(ctx, intr_name, tinfo->dst_type, tinfo->args, tinfo->arg_count,
+ return ac_emit_llvm_intrinsic(&ctx->ac, intr_name, tinfo->dst_type, tinfo->args, tinfo->arg_count,
AC_FUNC_ATTR_READNONE | AC_FUNC_ATTR_NOUNWIND);
}
@@ -1925,7 +1887,7 @@ static void visit_store_ssbo(struct nir_to_llvm_context *ctx,
}
params[0] = data;
params[3] = offset;
- emit_llvm_intrinsic(ctx, store_name,
+ ac_emit_llvm_intrinsic(&ctx->ac, store_name,
LLVMVoidTypeInContext(ctx->context), params, 6, 0);
}
}
@@ -1983,7 +1945,7 @@ static LLVMValueRef visit_atomic_ssbo(struct nir_to_llvm_context *ctx,
abort();
}
- return emit_llvm_intrinsic(ctx, name, ctx->i32, params, arg_count, 0);
+ return ac_emit_llvm_intrinsic(&ctx->ac, name, ctx->i32, params, arg_count, 0);
}
static LLVMValueRef visit_load_buffer(struct nir_to_llvm_context *ctx,
@@ -2014,7 +1976,7 @@ static LLVMValueRef visit_load_buffer(struct nir_to_llvm_context *ctx,
};
LLVMValueRef ret =
- emit_llvm_intrinsic(ctx, load_name, data_type, params, 5, 0);
+ ac_emit_llvm_intrinsic(&ctx->ac, load_name, data_type, params, 5, 0);
if (instr->num_components == 3)
ret = trim_vector(ctx, ret, 3);
@@ -2038,7 +2000,7 @@ static LLVMValueRef visit_load_ubo_buffer(struct nir_to_llvm_context *ctx,
LLVMBuildAdd(ctx->builder, LLVMConstInt(ctx->i32, 4 * i, 0),
offset, "")
};
- results[i] = emit_llvm_intrinsic(ctx, "llvm.SI.load.const", ctx->f32,
+ results[i] = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.load.const", ctx->f32,
params, 2, AC_FUNC_ATTR_READNONE);
}
@@ -2465,7 +2427,7 @@ static LLVMValueRef visit_image_load(struct nir_to_llvm_context *ctx,
params[2] = LLVMConstInt(ctx->i32, 0, false); /* voffset */
params[3] = LLVMConstInt(ctx->i1, 0, false); /* glc */
params[4] = LLVMConstInt(ctx->i1, 0, false); /* slc */
- res = emit_llvm_intrinsic(ctx, "llvm.amdgcn.buffer.load.format.v4f32", ctx->v4f32,
+ res = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.buffer.load.format.v4f32", ctx->v4f32,
params, 5, 0);
res = trim_vector(ctx, res, instr->dest.ssa.num_components);
@@ -2500,7 +2462,7 @@ static LLVMValueRef visit_image_load(struct nir_to_llvm_context *ctx,
LLVMTypeOf(params[1]), /* rsrc */
intrinsic_name, sizeof(intrinsic_name));
- res = emit_llvm_intrinsic(ctx, intrinsic_name, ctx->v4f32,
+ res = ac_emit_llvm_intrinsic(&ctx->ac, intrinsic_name, ctx->v4f32,
params, 7, AC_FUNC_ATTR_READONLY);
}
return to_integer(ctx, res);
@@ -2527,7 +2489,7 @@ static void visit_image_store(struct nir_to_llvm_context *ctx,
params[3] = LLVMConstInt(ctx->i32, 0, false); /* voffset */
params[4] = i1false; /* glc */
params[5] = i1false; /* slc */
- emit_llvm_intrinsic(ctx, "llvm.amdgcn.buffer.store.format.v4f32", ctx->voidt,
+ ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.buffer.store.format.v4f32", ctx->voidt,
params, 6, 0);
} else {
bool is_da = glsl_sampler_type_is_array(type) ||
@@ -2559,7 +2521,7 @@ static void visit_image_store(struct nir_to_llvm_context *ctx,
LLVMTypeOf(params[2]), /* rsrc */
intrinsic_name, sizeof(intrinsic_name));
- emit_llvm_intrinsic(ctx, intrinsic_name, ctx->voidt,
+ ac_emit_llvm_intrinsic(&ctx->ac, intrinsic_name, ctx->voidt,
params, 8, 0);
}
@@ -2637,7 +2599,7 @@ static LLVMValueRef visit_image_atomic(struct nir_to_llvm_context *ctx,
snprintf(intrinsic_name, sizeof(intrinsic_name),
"%s.%s.%s", base_name, atomic_name, coords_type);
- return emit_llvm_intrinsic(ctx, intrinsic_name, ctx->i32, params, param_count, 0);
+ return ac_emit_llvm_intrinsic(&ctx->ac, intrinsic_name, ctx->i32, params, param_count, 0);
}
static LLVMValueRef visit_image_size(struct nir_to_llvm_context *ctx,
@@ -2665,7 +2627,7 @@ static LLVMValueRef visit_image_size(struct nir_to_llvm_context *ctx,
params[8] = ctx->i32zero;
params[9] = ctx->i32zero;
- res = emit_llvm_intrinsic(ctx, "llvm.SI.getresinfo.i32", ctx->v4i32,
+ res = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.getresinfo.i32", ctx->v4i32,
params, 10, AC_FUNC_ATTR_READNONE);
if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
@@ -2684,14 +2646,14 @@ static void emit_waitcnt(struct nir_to_llvm_context *ctx)
LLVMValueRef args[1] = {
LLVMConstInt(ctx->i32, 0xf70, false),
};
- emit_llvm_intrinsic(ctx, "llvm.amdgcn.s.waitcnt",
+ ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.s.waitcnt",
ctx->voidt, args, 1, 0);
}
static void emit_barrier(struct nir_to_llvm_context *ctx)
{
// TODO tess
- emit_llvm_intrinsic(ctx, "llvm.amdgcn.s.barrier",
+ ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.s.barrier",
ctx->voidt, NULL, 0, 0);
}
@@ -2708,7 +2670,7 @@ static void emit_discard_if(struct nir_to_llvm_context *ctx,
cond = LLVMBuildSelect(ctx->builder, cond,
LLVMConstReal(ctx->f32, -1.0f),
ctx->f32zero, "");
- emit_llvm_intrinsic(ctx, "llvm.AMDGPU.kill",
+ ac_emit_llvm_intrinsic(&ctx->ac, "llvm.AMDGPU.kill",
LLVMVoidTypeInContext(ctx->context),
&cond, 1, 0);
}
@@ -2924,7 +2886,7 @@ static LLVMValueRef visit_interp(struct nir_to_llvm_context *ctx,
args[1] = attr_number;
args[2] = ctx->prim_mask;
args[3] = interp_param;
- result[chan] = emit_llvm_intrinsic(ctx, intr_name,
+ result[chan] = ac_emit_llvm_intrinsic(&ctx->ac, intr_name,
ctx->f32, args, args[3] ? 4 : 3,
AC_FUNC_ATTR_READNONE);
}
@@ -3035,7 +2997,7 @@ static void visit_intrinsic(struct nir_to_llvm_context *ctx,
break;
case nir_intrinsic_discard:
ctx->shader_info->fs.can_discard = true;
- emit_llvm_intrinsic(ctx, "llvm.AMDGPU.kilp",
+ ac_emit_llvm_intrinsic(&ctx->ac, "llvm.AMDGPU.kilp",
LLVMVoidTypeInContext(ctx->context),
NULL, 0, 0);
break;
@@ -3411,7 +3373,7 @@ static void visit_tex(struct nir_to_llvm_context *ctx, nir_tex_instr *instr)
/* This seems like a bit of a hack - but it passes Vulkan CTS with it */
if (instr->sampler_dim != GLSL_SAMPLER_DIM_3D && instr->op != nir_texop_txf) {
coords[2] = to_float(ctx, coords[2]);
- coords[2] = emit_llvm_intrinsic(ctx, "llvm.rint.f32", ctx->f32, &coords[2],
+ coords[2] = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.rint.f32", ctx->f32, &coords[2],
1, 0);
coords[2] = to_integer(ctx, coords[2]);
}
@@ -3793,7 +3755,7 @@ handle_vs_input_decl(struct nir_to_llvm_context *ctx,
args[0] = t_list;
args[1] = LLVMConstInt(ctx->i32, 0, false);
args[2] = buffer_index;
- input = emit_llvm_intrinsic(ctx,
+ input = ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.SI.vs.load.input", ctx->v4f32, args, 3,
AC_FUNC_ATTR_READNONE | AC_FUNC_ATTR_NOUNWIND);
@@ -3839,7 +3801,7 @@ static void interp_fs_input(struct nir_to_llvm_context *ctx,
args[1] = attr_number;
args[2] = prim_mask;
args[3] = interp_param;
- result[chan] = emit_llvm_intrinsic(ctx, intr_name,
+ result[chan] = ac_emit_llvm_intrinsic(&ctx->ac, intr_name,
ctx->f32, args, args[3] ? 4 : 3,
AC_FUNC_ATTR_READNONE | AC_FUNC_ATTR_NOUNWIND);
}
@@ -4107,7 +4069,7 @@ si_llvm_init_export_args(struct nir_to_llvm_context *ctx,
};
LLVMValueRef packed;
- packed = emit_llvm_intrinsic(ctx, "llvm.SI.packf16",
+ packed = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.packf16",
ctx->i32, pack_args, 2,
AC_FUNC_ATTR_READNONE);
args[chan + 5] = packed;
@@ -4275,7 +4237,7 @@ handle_vs_outputs_post(struct nir_to_llvm_context *ctx)
memcpy(pos_args[target - V_008DFC_SQ_EXP_POS],
args, sizeof(args));
} else {
- emit_llvm_intrinsic(ctx,
+ ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.SI.export",
LLVMVoidTypeInContext(ctx->context),
args, 9, 0);
@@ -4323,7 +4285,7 @@ handle_vs_outputs_post(struct nir_to_llvm_context *ctx)
pos_args[i][3] = LLVMConstInt(ctx->i32, V_008DFC_SQ_EXP_POS + pos_idx++, false);
if (pos_idx == num_pos_exports)
pos_args[i][2] = ctx->i32one;
- emit_llvm_intrinsic(ctx,
+ ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.SI.export",
LLVMVoidTypeInContext(ctx->context),
pos_args[i], 9, 0);
@@ -4348,7 +4310,7 @@ si_export_mrt_color(struct nir_to_llvm_context *ctx,
} else if (args[0] == ctx->i32zero)
return; /* unnecessary NULL export */
- emit_llvm_intrinsic(ctx, "llvm.SI.export",
+ ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.export",
ctx->voidt, args, 9, 0);
}
@@ -4392,7 +4354,7 @@ si_export_mrt_z(struct nir_to_llvm_context *ctx,
mask |= 0x01;
args[0] = LLVMConstInt(ctx->i32, mask, false);
- emit_llvm_intrinsic(ctx, "llvm.SI.export",
+ ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.export",
ctx->voidt, args, 9, 0);
}
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-26 18:34:40 +0000