diff options
author | Kenneth Graunke <[email protected]> | 2012-12-28 12:45:14 -0800 |
---|---|---|
committer | Kenneth Graunke <[email protected]> | 2013-01-03 13:36:04 -0800 |
commit | 82f8e8ebd57720f1e9d148c7dc65b14c218307df (patch) | |
tree | 5b27bf47cbab91f7ce66779b2f11f59e388a0a7d /src | |
parent | 5bf357db89836d0f4e4b8a4cb559755d4734b81b (diff) |
i965: Replace structs with bit-shifting for Gen7 SURFACE_STATE entries.
Every generation except Gen7 creates SURFACE_STATE entries via a
uint32_t array. Only Gen7 uses the older bitfield structure, which we
moved away from because it was less efficient. Convert it for
consistency.
This reduces the compiled size of gen7_wm_surface_state.o by 2.86% in a
release build.
v2: Fix accidental use of BRW_SURFACE_WIDTH/HEIGHT in brw_state_dump.c;
switch back to gen7_set_surface_mcs_info setting surf[6] directly
(both per Eric's review comments).
Acked-by: Ian Romanick <[email protected]>
Reviewed-by: Eric Anholt <[email protected]>
Diffstat (limited to 'src')
-rw-r--r-- | src/mesa/drivers/dri/i965/brw_defines.h | 43 | ||||
-rw-r--r-- | src/mesa/drivers/dri/i965/brw_state.h | 11 | ||||
-rw-r--r-- | src/mesa/drivers/dri/i965/brw_state_dump.c | 22 | ||||
-rw-r--r-- | src/mesa/drivers/dri/i965/brw_structs.h | 102 | ||||
-rw-r--r-- | src/mesa/drivers/dri/i965/gen7_blorp.cpp | 65 | ||||
-rw-r--r-- | src/mesa/drivers/dri/i965/gen7_wm_surface_state.c | 369 |
6 files changed, 252 insertions, 360 deletions
diff --git a/src/mesa/drivers/dri/i965/brw_defines.h b/src/mesa/drivers/dri/i965/brw_defines.h index ab206d1920f..1d0cf0275d1 100644 --- a/src/mesa/drivers/dri/i965/brw_defines.h +++ b/src/mesa/drivers/dri/i965/brw_defines.h @@ -439,8 +439,16 @@ #define BRW_SURFACE_BUFFER 4 #define BRW_SURFACE_NULL 7 -#define GEN7_SURFACE_ARYSPC_FULL 0 -#define GEN7_SURFACE_ARYSPC_LOD0 1 +#define GEN7_SURFACE_IS_ARRAY (1 << 28) +#define GEN7_SURFACE_VALIGN_2 (0 << 16) +#define GEN7_SURFACE_VALIGN_4 (1 << 16) +#define GEN7_SURFACE_HALIGN_4 (0 << 15) +#define GEN7_SURFACE_HALIGN_8 (1 << 15) +#define GEN7_SURFACE_TILING_NONE (0 << 13) +#define GEN7_SURFACE_TILING_X (2 << 13) +#define GEN7_SURFACE_TILING_Y (3 << 13) +#define GEN7_SURFACE_ARYSPC_FULL (0 << 10) +#define GEN7_SURFACE_ARYSPC_LOD0 (1 << 10) /* Surface state DW2 */ #define BRW_SURFACE_HEIGHT_SHIFT 19 @@ -449,6 +457,10 @@ #define BRW_SURFACE_WIDTH_MASK INTEL_MASK(18, 6) #define BRW_SURFACE_LOD_SHIFT 2 #define BRW_SURFACE_LOD_MASK INTEL_MASK(5, 2) +#define GEN7_SURFACE_HEIGHT_SHIFT 16 +#define GEN7_SURFACE_HEIGHT_MASK INTEL_MASK(29, 16) +#define GEN7_SURFACE_WIDTH_SHIFT 0 +#define GEN7_SURFACE_WIDTH_MASK INTEL_MASK(13, 0) /* Surface state DW3 */ #define BRW_SURFACE_DEPTH_SHIFT 21 @@ -463,11 +475,11 @@ #define BRW_SURFACE_MIN_LOD_MASK INTEL_MASK(31, 28) #define BRW_SURFACE_MULTISAMPLECOUNT_1 (0 << 4) #define BRW_SURFACE_MULTISAMPLECOUNT_4 (2 << 4) -#define GEN7_SURFACE_MULTISAMPLECOUNT_1 0 -#define GEN7_SURFACE_MULTISAMPLECOUNT_4 2 -#define GEN7_SURFACE_MULTISAMPLECOUNT_8 3 -#define GEN7_SURFACE_MSFMT_MSS 0 -#define GEN7_SURFACE_MSFMT_DEPTH_STENCIL 1 +#define GEN7_SURFACE_MULTISAMPLECOUNT_1 (0 << 3) +#define GEN7_SURFACE_MULTISAMPLECOUNT_4 (2 << 3) +#define GEN7_SURFACE_MULTISAMPLECOUNT_8 (3 << 3) +#define GEN7_SURFACE_MSFMT_MSS (0 << 6) +#define GEN7_SURFACE_MSFMT_DEPTH_STENCIL (1 << 6) /* Surface state DW5 */ #define BRW_SURFACE_X_OFFSET_SHIFT 25 @@ -475,8 +487,25 @@ #define BRW_SURFACE_VERTICAL_ALIGN_ENABLE (1 << 24) #define BRW_SURFACE_Y_OFFSET_SHIFT 20 #define BRW_SURFACE_Y_OFFSET_MASK INTEL_MASK(23, 20) +#define GEN7_SURFACE_MIN_LOD_SHIFT 4 +#define GEN7_SURFACE_MIN_LOD_MASK INTEL_MASK(7, 4) + +/* Surface state DW6 */ +#define GEN7_SURFACE_MCS_ENABLE (1 << 0) +#define GEN7_SURFACE_MCS_PITCH_SHIFT 3 +#define GEN7_SURFACE_MCS_PITCH_MASK INTEL_MASK(11, 3) /* Surface state DW7 */ +#define GEN7_SURFACE_SCS_R_SHIFT 25 +#define GEN7_SURFACE_SCS_R_MASK INTEL_MASK(27, 25) +#define GEN7_SURFACE_SCS_G_SHIFT 22 +#define GEN7_SURFACE_SCS_G_MASK INTEL_MASK(24, 22) +#define GEN7_SURFACE_SCS_B_SHIFT 19 +#define GEN7_SURFACE_SCS_B_MASK INTEL_MASK(21, 19) +#define GEN7_SURFACE_SCS_A_SHIFT 16 +#define GEN7_SURFACE_SCS_A_MASK INTEL_MASK(18, 16) + +/* The actual swizzle values/what channel to use */ #define HSW_SCS_ZERO 0 #define HSW_SCS_ONE 1 #define HSW_SCS_RED 4 diff --git a/src/mesa/drivers/dri/i965/brw_state.h b/src/mesa/drivers/dri/i965/brw_state.h index deaef0e157a..c6b8219e6bb 100644 --- a/src/mesa/drivers/dri/i965/brw_state.h +++ b/src/mesa/drivers/dri/i965/brw_state.h @@ -205,17 +205,14 @@ GLuint translate_tex_format(gl_format mesa_format, int brw_get_texture_swizzle(const struct gl_texture_object *t); /* gen7_wm_surface_state.c */ -void gen7_set_surface_tiling(struct gen7_surface_state *surf, uint32_t tiling); -void gen7_set_surface_msaa(struct gen7_surface_state *surf, - unsigned num_samples, - enum intel_msaa_layout layout); +uint32_t gen7_surface_tiling_mode(uint32_t tiling); +uint32_t gen7_surface_msaa_bits(unsigned num_samples, enum intel_msaa_layout l); void gen7_set_surface_mcs_info(struct brw_context *brw, - struct gen7_surface_state *surf, + uint32_t *surf, uint32_t surf_offset, const struct intel_mipmap_tree *mcs_mt, bool is_render_target); -void gen7_check_surface_setup(struct gen7_surface_state *surf, - bool is_render_target); +void gen7_check_surface_setup(uint32_t *surf, bool is_render_target); void gen7_init_vtable_surface_functions(struct brw_context *brw); /* brw_wm_sampler_state.c */ diff --git a/src/mesa/drivers/dri/i965/brw_state_dump.c b/src/mesa/drivers/dri/i965/brw_state_dump.c index 7306ceb969d..9ea3fac1536 100644 --- a/src/mesa/drivers/dri/i965/brw_state_dump.c +++ b/src/mesa/drivers/dri/i965/brw_state_dump.c @@ -184,8 +184,8 @@ static void dump_surface_state(struct brw_context *brw, uint32_t offset) get_965_surface_format(GET_FIELD(surf[0], BRW_SURFACE_FORMAT))); batch_out(brw, name, offset, 1, "offset\n"); batch_out(brw, name, offset, 2, "%dx%d size, %d mips\n", - GET_FIELD(surf[2], BRW_SURFACE_WIDTH) + 1, - GET_FIELD(surf[2], BRW_SURFACE_HEIGHT) + 1, + GET_FIELD(surf[2], GEN7_SURFACE_WIDTH) + 1, + GET_FIELD(surf[2], GEN7_SURFACE_HEIGHT) + 1, GET_FIELD(surf[2], BRW_SURFACE_LOD)); batch_out(brw, name, offset, 3, "pitch %d, %s tiled\n", GET_FIELD(surf[3], BRW_SURFACE_PITCH) + 1, @@ -201,20 +201,24 @@ static void dump_surface_state(struct brw_context *brw, uint32_t offset) static void dump_gen7_surface_state(struct brw_context *brw, uint32_t offset) { const char *name = "SURF"; - struct gen7_surface_state *surf = brw->intel.batch.bo->virtual + offset; + uint32_t *surf = brw->intel.batch.bo->virtual + offset; batch_out(brw, name, offset, 0, "%s %s\n", - get_965_surfacetype(surf->ss0.surface_type), - get_965_surface_format(surf->ss0.surface_format)); + get_965_surfacetype(GET_FIELD(surf[0], BRW_SURFACE_TYPE)), + get_965_surface_format(GET_FIELD(surf[0], BRW_SURFACE_FORMAT))); batch_out(brw, name, offset, 1, "offset\n"); batch_out(brw, name, offset, 2, "%dx%d size, %d mips\n", - surf->ss2.width + 1, surf->ss2.height + 1, surf->ss5.mip_count); + GET_FIELD(surf[2], BRW_SURFACE_WIDTH) + 1, + GET_FIELD(surf[2], BRW_SURFACE_HEIGHT) + 1, + surf[5] & INTEL_MASK(3, 0)); batch_out(brw, name, offset, 3, "pitch %d, %stiled\n", - surf->ss3.pitch + 1, surf->ss0.tiled_surface ? "" : "not "); + (surf[3] & INTEL_MASK(17, 0)) + 1, + (surf[0] & (1 << 14)) ? "" : "not "); batch_out(brw, name, offset, 4, "mip base %d\n", - surf->ss5.min_lod); + GET_FIELD(surf[5], GEN7_SURFACE_MIN_LOD)); batch_out(brw, name, offset, 5, "x,y offset: %d,%d\n", - surf->ss5.x_offset, surf->ss5.y_offset); + GET_FIELD(surf[5], BRW_SURFACE_X_OFFSET), + GET_FIELD(surf[5], BRW_SURFACE_Y_OFFSET)); } static void diff --git a/src/mesa/drivers/dri/i965/brw_structs.h b/src/mesa/drivers/dri/i965/brw_structs.h index a72e9448f24..75365e07566 100644 --- a/src/mesa/drivers/dri/i965/brw_structs.h +++ b/src/mesa/drivers/dri/i965/brw_structs.h @@ -774,108 +774,6 @@ struct gen7_sf_clip_viewport { GLfloat pad1[4]; }; -/* volume 5c Shared Functions - 1.13.4.1.2 */ -struct gen7_surface_state -{ - struct { - GLuint cube_pos_z:1; - GLuint cube_neg_z:1; - GLuint cube_pos_y:1; - GLuint cube_neg_y:1; - GLuint cube_pos_x:1; - GLuint cube_neg_x:1; - GLuint pad2:2; - GLuint render_cache_read_write:1; - GLuint pad1:1; - GLuint surface_array_spacing:1; - GLuint vert_line_stride_ofs:1; - GLuint vert_line_stride:1; - GLuint tile_walk:1; - GLuint tiled_surface:1; - GLuint horizontal_alignment:1; - GLuint vertical_alignment:2; - GLuint surface_format:9; /**< BRW_SURFACEFORMAT_x */ - GLuint pad0:1; - GLuint is_array:1; - GLuint surface_type:3; /**< BRW_SURFACE_1D/2D/3D/CUBE */ - } ss0; - - struct { - GLuint base_addr; - } ss1; - - struct { - GLuint width:14; - GLuint pad1:2; - GLuint height:14; - GLuint pad0:2; - } ss2; - - struct { - GLuint pitch:18; - GLuint pad:3; - GLuint depth:11; - } ss3; - - struct { - GLuint multisample_position_palette_index:3; - GLuint num_multisamples:3; - GLuint multisampled_surface_storage_format:1; - GLuint render_target_view_extent:11; - GLuint min_array_elt:11; - GLuint rotation:2; - GLuint pad0:1; - } ss4; - - struct { - GLuint mip_count:4; - GLuint min_lod:4; - GLuint pad1:12; - GLuint y_offset:4; - GLuint pad0:1; - GLuint x_offset:7; - } ss5; - - union { - GLuint raw_data; - struct { - GLuint y_offset_for_uv_plane:14; - GLuint pad1:2; - GLuint x_offset_for_uv_plane:14; - GLuint pad0:2; - } planar; /** Interpretation when Surface Format == PLANAR */ - struct { - GLuint mcs_enable:1; - GLuint append_counter_enable:1; - GLuint pad:4; - GLuint append_counter_address:26; - } mcs_disabled; /** Interpretation when mcs_enable == 0 */ - struct { - GLuint mcs_enable:1; - GLuint pad:2; - GLuint mcs_surface_pitch:9; - GLuint mcs_base_address:20; - } mcs_enabled; /** Interpretation when mcs_enable == 1 */ - } ss6; - - struct { - GLuint resource_min_lod:12; - - /* Only on Haswell */ - GLuint pad0:4; - GLuint shader_channel_select_a:3; - GLuint shader_channel_select_b:3; - GLuint shader_channel_select_g:3; - GLuint shader_channel_select_r:3; - - GLuint alpha_clear_color:1; - GLuint blue_clear_color:1; - GLuint green_clear_color:1; - GLuint red_clear_color:1; - } ss7; -}; - - struct brw_vertex_element_state { struct diff --git a/src/mesa/drivers/dri/i965/gen7_blorp.cpp b/src/mesa/drivers/dri/i965/gen7_blorp.cpp index 317527396e3..e30f6ef8f16 100644 --- a/src/mesa/drivers/dri/i965/gen7_blorp.cpp +++ b/src/mesa/drivers/dri/i965/gen7_blorp.cpp @@ -152,64 +152,65 @@ gen7_blorp_emit_surface_state(struct brw_context *brw, struct intel_region *region = surface->mt->region; uint32_t tile_x, tile_y; - struct gen7_surface_state *surf = (struct gen7_surface_state *) - brw_state_batch(brw, AUB_TRACE_SURFACE_STATE, sizeof(*surf), 32, - &wm_surf_offset); - memset(surf, 0, sizeof(*surf)); + uint32_t tiling = surface->map_stencil_as_y_tiled + ? I915_TILING_Y : region->tiling; + + uint32_t *surf = (uint32_t *) + brw_state_batch(brw, AUB_TRACE_SURFACE_STATE, 8 * 4, 32, &wm_surf_offset); + memset(surf, 0, 8 * 4); + + surf[0] = BRW_SURFACE_2D << BRW_SURFACE_TYPE_SHIFT | + surface->brw_surfaceformat << BRW_SURFACE_FORMAT_SHIFT | + gen7_surface_tiling_mode(tiling); if (surface->mt->align_h == 4) - surf->ss0.vertical_alignment = 1; + surf[0] |= GEN7_SURFACE_VALIGN_4; if (surface->mt->align_w == 8) - surf->ss0.horizontal_alignment = 1; + surf[0] |= GEN7_SURFACE_HALIGN_8; - surf->ss0.surface_format = surface->brw_surfaceformat; - surf->ss0.surface_type = BRW_SURFACE_2D; - surf->ss0.surface_array_spacing = surface->array_spacing_lod0 ? - GEN7_SURFACE_ARYSPC_LOD0 : GEN7_SURFACE_ARYSPC_FULL; + if (surface->array_spacing_lod0) + surf[0] |= GEN7_SURFACE_ARYSPC_LOD0; + else + surf[0] |= GEN7_SURFACE_ARYSPC_FULL; /* reloc */ - surf->ss1.base_addr = surface->compute_tile_offsets(&tile_x, &tile_y); - surf->ss1.base_addr += region->bo->offset; + surf[1] = + surface->compute_tile_offsets(&tile_x, &tile_y) + region->bo->offset; /* Note that the low bits of these fields are missing, so * there's the possibility of getting in trouble. */ assert(tile_x % 4 == 0); assert(tile_y % 2 == 0); - surf->ss5.x_offset = tile_x / 4; - surf->ss5.y_offset = tile_y / 2; - - surf->ss2.width = width - 1; - surf->ss2.height = height - 1; + surf[5] = SET_FIELD(tile_x / 4, BRW_SURFACE_X_OFFSET) | + SET_FIELD(tile_y / 2, BRW_SURFACE_Y_OFFSET); - uint32_t tiling = surface->map_stencil_as_y_tiled - ? I915_TILING_Y : region->tiling; - gen7_set_surface_tiling(surf, tiling); + surf[2] = SET_FIELD(width - 1, GEN7_SURFACE_WIDTH) | + SET_FIELD(height - 1, GEN7_SURFACE_HEIGHT); uint32_t pitch_bytes = region->pitch * region->cpp; if (surface->map_stencil_as_y_tiled) pitch_bytes *= 2; - surf->ss3.pitch = pitch_bytes - 1; + surf[3] = pitch_bytes - 1; - gen7_set_surface_msaa(surf, surface->num_samples, surface->msaa_layout); + surf[4] = gen7_surface_msaa_bits(surface->num_samples, surface->msaa_layout); if (surface->msaa_layout == INTEL_MSAA_LAYOUT_CMS) { - gen7_set_surface_mcs_info(brw, surf, wm_surf_offset, - surface->mt->mcs_mt, is_render_target); + gen7_set_surface_mcs_info(brw, surf, wm_surf_offset, surface->mt->mcs_mt, + is_render_target); } if (intel->is_haswell) { - surf->ss7.shader_channel_select_r = HSW_SCS_RED; - surf->ss7.shader_channel_select_g = HSW_SCS_GREEN; - surf->ss7.shader_channel_select_b = HSW_SCS_BLUE; - surf->ss7.shader_channel_select_a = HSW_SCS_ALPHA; + surf[7] = SET_FIELD(HSW_SCS_RED, GEN7_SURFACE_SCS_R) | + SET_FIELD(HSW_SCS_GREEN, GEN7_SURFACE_SCS_G) | + SET_FIELD(HSW_SCS_BLUE, GEN7_SURFACE_SCS_B) | + SET_FIELD(HSW_SCS_ALPHA, GEN7_SURFACE_SCS_A); } /* Emit relocation to surface contents */ - drm_intel_bo_emit_reloc(brw->intel.batch.bo, - wm_surf_offset + - offsetof(struct gen7_surface_state, ss1), + drm_intel_bo_emit_reloc(intel->batch.bo, + wm_surf_offset + 4, region->bo, - surf->ss1.base_addr - region->bo->offset, + surf[1] - region->bo->offset, read_domains, write_domain); gen7_check_surface_setup(surf, is_render_target); diff --git a/src/mesa/drivers/dri/i965/gen7_wm_surface_state.c b/src/mesa/drivers/dri/i965/gen7_wm_surface_state.c index 1e5af959512..13e8ba8ecdc 100644 --- a/src/mesa/drivers/dri/i965/gen7_wm_surface_state.c +++ b/src/mesa/drivers/dri/i965/gen7_wm_surface_state.c @@ -61,47 +61,44 @@ swizzle_to_scs(GLenum swizzle) return HSW_SCS_ZERO; } -void -gen7_set_surface_tiling(struct gen7_surface_state *surf, uint32_t tiling) +uint32_t +gen7_surface_tiling_mode(uint32_t tiling) { switch (tiling) { - case I915_TILING_NONE: - surf->ss0.tiled_surface = 0; - surf->ss0.tile_walk = 0; - break; case I915_TILING_X: - surf->ss0.tiled_surface = 1; - surf->ss0.tile_walk = BRW_TILEWALK_XMAJOR; - break; + return GEN7_SURFACE_TILING_X; case I915_TILING_Y: - surf->ss0.tiled_surface = 1; - surf->ss0.tile_walk = BRW_TILEWALK_YMAJOR; - break; + return GEN7_SURFACE_TILING_Y; + default: + return GEN7_SURFACE_TILING_NONE; } } -void -gen7_set_surface_msaa(struct gen7_surface_state *surf, unsigned num_samples, - enum intel_msaa_layout layout) +uint32_t +gen7_surface_msaa_bits(unsigned num_samples, enum intel_msaa_layout layout) { + uint32_t ss4 = 0; + if (num_samples > 4) - surf->ss4.num_multisamples = GEN7_SURFACE_MULTISAMPLECOUNT_8; + ss4 |= GEN7_SURFACE_MULTISAMPLECOUNT_8; else if (num_samples > 1) - surf->ss4.num_multisamples = GEN7_SURFACE_MULTISAMPLECOUNT_4; + ss4 |= GEN7_SURFACE_MULTISAMPLECOUNT_4; + else + ss4 |= GEN7_SURFACE_MULTISAMPLECOUNT_1; + + if (layout == INTEL_MSAA_LAYOUT_IMS) + ss4 |= GEN7_SURFACE_MSFMT_DEPTH_STENCIL; else - surf->ss4.num_multisamples = GEN7_SURFACE_MULTISAMPLECOUNT_1; + ss4 |= GEN7_SURFACE_MSFMT_MSS; - surf->ss4.multisampled_surface_storage_format = - layout == INTEL_MSAA_LAYOUT_IMS ? - GEN7_SURFACE_MSFMT_DEPTH_STENCIL : - GEN7_SURFACE_MSFMT_MSS; + return ss4; } void gen7_set_surface_mcs_info(struct brw_context *brw, - struct gen7_surface_state *surf, + uint32_t *surf, uint32_t surf_offset, const struct intel_mipmap_tree *mcs_mt, bool is_render_target) @@ -125,14 +122,15 @@ gen7_set_surface_mcs_info(struct brw_context *brw, * the necessary address translation. */ assert ((mcs_mt->region->bo->offset & 0xfff) == 0); - surf->ss6.mcs_enabled.mcs_enable = 1; - surf->ss6.mcs_enabled.mcs_surface_pitch = pitch_tiles - 1; - surf->ss6.mcs_enabled.mcs_base_address = mcs_mt->region->bo->offset >> 12; + + surf[6] = GEN7_SURFACE_MCS_ENABLE | + SET_FIELD(pitch_tiles - 1, GEN7_SURFACE_MCS_PITCH) | + mcs_mt->region->bo->offset; + drm_intel_bo_emit_reloc(brw->intel.batch.bo, - surf_offset + - offsetof(struct gen7_surface_state, ss6), + surf_offset + 6 * 4, mcs_mt->region->bo, - surf->ss6.raw_data & 0xfff, + surf[6] & 0xfff, is_render_target ? I915_GEM_DOMAIN_RENDER : I915_GEM_DOMAIN_SAMPLER, is_render_target ? I915_GEM_DOMAIN_RENDER : 0); @@ -140,11 +138,15 @@ gen7_set_surface_mcs_info(struct brw_context *brw, void -gen7_check_surface_setup(struct gen7_surface_state *surf, - bool is_render_target) +gen7_check_surface_setup(uint32_t *surf, bool is_render_target) { - bool is_multisampled = - surf->ss4.num_multisamples != GEN7_SURFACE_MULTISAMPLECOUNT_1; + unsigned num_multisamples = surf[4] & INTEL_MASK(5, 3); + unsigned multisampled_surface_storage_format = surf[4] & (1 << 6); + unsigned surface_array_spacing = surf[0] & (1 << 10); + bool is_multisampled = num_multisamples != GEN7_SURFACE_MULTISAMPLECOUNT_1; + + (void) surface_array_spacing; + /* From the Graphics BSpec: vol5c Shared Functions [SNB+] > State > * SURFACE_STATE > SURFACE_STATE for most messages [DevIVB]: Surface Array * Spacing: @@ -153,9 +155,9 @@ gen7_check_surface_setup(struct gen7_surface_state *surf, * Multisamples is not MULTISAMPLECOUNT_1, this field must be set to * ARYSPC_LOD0. */ - if (surf->ss4.multisampled_surface_storage_format == GEN7_SURFACE_MSFMT_MSS + if (multisampled_surface_storage_format == GEN7_SURFACE_MSFMT_MSS && is_multisampled) - assert(surf->ss0.surface_array_spacing == GEN7_SURFACE_ARYSPC_LOD0); + assert(surface_array_spacing == GEN7_SURFACE_ARYSPC_LOD0); /* From the Graphics BSpec: vol5c Shared Functions [SNB+] > State > * SURFACE_STATE > SURFACE_STATE for most messages [DevIVB]: Multisampled @@ -169,8 +171,7 @@ gen7_check_surface_setup(struct gen7_surface_state *surf, * This field is ignored if Number of Multisamples is MULTISAMPLECOUNT_1. */ if (is_render_target && is_multisampled) { - assert(surf->ss4.multisampled_surface_storage_format == - GEN7_SURFACE_MSFMT_MSS); + assert(multisampled_surface_storage_format == GEN7_SURFACE_MSFMT_MSS); } /* From the Graphics BSpec: vol5c Shared Functions [SNB+] > State > @@ -181,10 +182,9 @@ gen7_check_surface_setup(struct gen7_surface_state *surf, * is >= 8192 (meaning the actual surface width is >= 8193 pixels), this * field must be set to MSFMT_MSS. */ - if (surf->ss4.num_multisamples == GEN7_SURFACE_MULTISAMPLECOUNT_8 && - surf->ss2.width >= 8192) { - assert(surf->ss4.multisampled_surface_storage_format == - GEN7_SURFACE_MSFMT_MSS); + uint32_t width = GET_FIELD(surf[2], GEN7_SURFACE_WIDTH) + 1; + if (num_multisamples == GEN7_SURFACE_MULTISAMPLECOUNT_8 && width >= 8193) { + assert(multisampled_surface_storage_format == GEN7_SURFACE_MSFMT_MSS); } /* From the Graphics BSpec: vol5c Shared Functions [SNB+] > State > @@ -203,25 +203,25 @@ gen7_check_surface_setup(struct gen7_surface_state *surf, * * This field is ignored if Number of Multisamples is MULTISAMPLECOUNT_1. */ - uint32_t depth = surf->ss3.depth + 1; - uint32_t height = surf->ss2.height + 1; - if (surf->ss4.num_multisamples == GEN7_SURFACE_MULTISAMPLECOUNT_8 && + uint32_t depth = GET_FIELD(surf[3], BRW_SURFACE_DEPTH) + 1; + uint32_t height = GET_FIELD(surf[2], GEN7_SURFACE_HEIGHT) + 1; + if (num_multisamples == GEN7_SURFACE_MULTISAMPLECOUNT_8 && depth * height > 4194304) { - assert(surf->ss4.multisampled_surface_storage_format == + assert(multisampled_surface_storage_format == GEN7_SURFACE_MSFMT_DEPTH_STENCIL); } - if (surf->ss4.num_multisamples == GEN7_SURFACE_MULTISAMPLECOUNT_4 && + if (num_multisamples == GEN7_SURFACE_MULTISAMPLECOUNT_4 && depth * height > 8388608) { - assert(surf->ss4.multisampled_surface_storage_format == + assert(multisampled_surface_storage_format == GEN7_SURFACE_MSFMT_DEPTH_STENCIL); } if (is_multisampled) { - switch (surf->ss0.surface_format) { + switch (GET_FIELD(surf[0], BRW_SURFACE_FORMAT)) { case BRW_SURFACEFORMAT_I24X8_UNORM: case BRW_SURFACEFORMAT_L24X8_UNORM: case BRW_SURFACEFORMAT_A24X8_UNORM: case BRW_SURFACEFORMAT_R24_UNORM_X8_TYPELESS: - assert(surf->ss4.multisampled_surface_storage_format == + assert(multisampled_surface_storage_format == GEN7_SURFACE_MSFMT_DEPTH_STENCIL); } } @@ -235,56 +235,47 @@ gen7_update_buffer_texture_surface(struct gl_context *ctx, unsigned surf_index) { struct brw_context *brw = brw_context(ctx); + struct intel_context *intel = &brw->intel; struct gl_texture_object *tObj = ctx->Texture.Unit[unit]._Current; - struct gen7_surface_state *surf; struct intel_buffer_object *intel_obj = intel_buffer_object(tObj->BufferObject); drm_intel_bo *bo = intel_obj ? intel_obj->buffer : NULL; gl_format format = tObj->_BufferObjectFormat; - int texel_size = _mesa_get_format_bytes(format); - - surf = brw_state_batch(brw, AUB_TRACE_SURFACE_STATE, - sizeof(*surf), 32, &binding_table[surf_index]); - memset(surf, 0, sizeof(*surf)); - surf->ss0.surface_type = BRW_SURFACE_BUFFER; - surf->ss0.surface_format = brw_format_for_mesa_format(format); + uint32_t *surf = brw_state_batch(brw, AUB_TRACE_SURFACE_STATE, + 8 * 4, 32, &binding_table[surf_index]); + memset(surf, 0, 8 * 4); - surf->ss0.render_cache_read_write = 1; - - if (surf->ss0.surface_format == 0 && format != MESA_FORMAT_RGBA_FLOAT32) { + uint32_t surface_format = brw_format_for_mesa_format(format); + if (surface_format == 0 && format != MESA_FORMAT_RGBA_FLOAT32) { _mesa_problem(NULL, "bad format %s for texture buffer\n", - _mesa_get_format_name(format)); + _mesa_get_format_name(format)); } + surf[0] = BRW_SURFACE_BUFFER << BRW_SURFACE_TYPE_SHIFT | + surface_format << BRW_SURFACE_FORMAT_SHIFT | + BRW_SURFACE_RC_READ_WRITE; + if (bo) { - surf->ss1.base_addr = bo->offset; /* reloc */ + surf[1] = bo->offset; /* reloc */ /* Emit relocation to surface contents. Section 5.1.1 of the gen4 * bspec ("Data Cache") says that the data cache does not exist as * a separate cache and is just the sampler cache. */ - drm_intel_bo_emit_reloc(brw->intel.batch.bo, - (binding_table[surf_index] + - offsetof(struct gen7_surface_state, ss1)), + drm_intel_bo_emit_reloc(intel->batch.bo, + binding_table[surf_index] + 4, bo, 0, I915_GEM_DOMAIN_SAMPLER, 0); + int texel_size = _mesa_get_format_bytes(format); int w = intel_obj->Base.Size / texel_size; - surf->ss2.width = w & 0x7f; /* bits 6:0 of size or width */ - surf->ss2.height = (w >> 7) & 0x1fff; /* bits 19:7 of size or width */ - surf->ss3.depth = (w >> 20) & 0x7f; /* bits 26:20 of size or width */ - surf->ss3.pitch = texel_size - 1; -} else { - surf->ss1.base_addr = 0; - surf->ss2.width = 0; - surf->ss2.height = 0; - surf->ss3.depth = 0; - surf->ss3.pitch = 0; + surf[2] = SET_FIELD(w & 0x7f, GEN7_SURFACE_WIDTH) | /* bits 6:0 of size */ + SET_FIELD((w >> 7) & 0x1fff, GEN7_SURFACE_HEIGHT); /* 19:7 */ + surf[3] = SET_FIELD((w >> 20) & 0x7f, BRW_SURFACE_DEPTH) | /* bits 26:20 */ + (texel_size - 1); } - gen7_set_surface_tiling(surf, I915_TILING_NONE); - gen7_check_surface_setup(surf, false /* is_render_target */); } @@ -295,12 +286,12 @@ gen7_update_texture_surface(struct gl_context *ctx, unsigned surf_index) { struct brw_context *brw = brw_context(ctx); + struct intel_context *intel = &brw->intel; struct gl_texture_object *tObj = ctx->Texture.Unit[unit]._Current; struct intel_texture_object *intelObj = intel_texture_object(tObj); struct intel_mipmap_tree *mt = intelObj->mt; struct gl_texture_image *firstImage = tObj->Image[0][tObj->BaseLevel]; struct gl_sampler_object *sampler = _mesa_get_samplerobj(ctx, unit); - struct gen7_surface_state *surf; int width, height, depth; if (tObj->Target == GL_TEXTURE_BUFFER) { @@ -314,62 +305,38 @@ gen7_update_texture_surface(struct gl_context *ctx, intel_miptree_get_dimensions_for_image(firstImage, &width, &height, &depth); - surf = brw_state_batch(brw, AUB_TRACE_SURFACE_STATE, - sizeof(*surf), 32, &binding_table[surf_index]); - memset(surf, 0, sizeof(*surf)); - - if (mt->align_h == 4) - surf->ss0.vertical_alignment = 1; - if (mt->align_w == 8) - surf->ss0.horizontal_alignment = 1; - - surf->ss0.surface_type = translate_tex_target(tObj->Target); - surf->ss0.surface_format = translate_tex_format(mt->format, - firstImage->InternalFormat, - tObj->DepthMode, - sampler->sRGBDecode); - if (tObj->Target == GL_TEXTURE_CUBE_MAP || - tObj->Target == GL_TEXTURE_CUBE_MAP_ARRAY) { - surf->ss0.cube_pos_x = 1; - surf->ss0.cube_pos_y = 1; - surf->ss0.cube_pos_z = 1; - surf->ss0.cube_neg_x = 1; - surf->ss0.cube_neg_y = 1; - surf->ss0.cube_neg_z = 1; - } - - surf->ss0.is_array = depth > 1 && tObj->Target != GL_TEXTURE_3D; + uint32_t *surf = brw_state_batch(brw, AUB_TRACE_SURFACE_STATE, + 8 * 4, 32, &binding_table[surf_index]); + memset(surf, 0, 8 * 4); - gen7_set_surface_tiling(surf, intelObj->mt->region->tiling); + uint32_t tex_format = translate_tex_format(mt->format, + firstImage->InternalFormat, + tObj->DepthMode, + sampler->sRGBDecode); - /* ss0 remaining fields: - * - vert_line_stride (exists on gen6 but we ignore it) - * - vert_line_stride_ofs (exists on gen6 but we ignore it) - * - surface_array_spacing - * - render_cache_read_write (exists on gen6 but ignored here) - */ - - surf->ss1.base_addr = - intelObj->mt->region->bo->offset + intelObj->mt->offset; /* reloc */ + surf[0] = translate_tex_target(tObj->Target) << BRW_SURFACE_TYPE_SHIFT | + tex_format << BRW_SURFACE_FORMAT_SHIFT | + gen7_surface_tiling_mode(mt->region->tiling) | + BRW_SURFACE_CUBEFACE_ENABLES; - surf->ss2.width = width - 1; - surf->ss2.height = height - 1; + if (mt->align_h == 4) + surf[0] |= GEN7_SURFACE_VALIGN_4; + if (mt->align_w == 8) + surf[0] |= GEN7_SURFACE_HALIGN_8; - surf->ss3.pitch = (intelObj->mt->region->pitch * intelObj->mt->cpp) - 1; - surf->ss3.depth = depth - 1; + if (depth > 1 && tObj->Target != GL_TEXTURE_3D) + surf[0] |= GEN7_SURFACE_IS_ARRAY; - /* ss4: ignored? */ + surf[1] = mt->region->bo->offset + mt->offset; /* reloc */ - surf->ss5.mip_count = intelObj->_MaxLevel - tObj->BaseLevel; - surf->ss5.min_lod = 0; + surf[2] = SET_FIELD(width - 1, GEN7_SURFACE_WIDTH) | + SET_FIELD(height - 1, GEN7_SURFACE_HEIGHT); + surf[3] = SET_FIELD(depth - 1, BRW_SURFACE_DEPTH) | + ((intelObj->mt->region->pitch * intelObj->mt->cpp) - 1); - /* ss5 remaining fields: - * - x_offset (N/A for textures?) - * - y_offset (ditto) - * - cache_control - */ + surf[5] = intelObj->_MaxLevel - tObj->BaseLevel; /* mip count */ - if (brw->intel.is_haswell) { + if (intel->is_haswell) { /* Handling GL_ALPHA as a surface format override breaks 1.30+ style * texturing functions that return a float, as our code generation always * selects the .x channel (which would always be 0). @@ -381,16 +348,17 @@ gen7_update_texture_surface(struct gl_context *ctx, const int swizzle = unlikely(alpha_depth) ? SWIZZLE_XYZW : brw_get_texture_swizzle(tObj); - surf->ss7.shader_channel_select_r = swizzle_to_scs(GET_SWZ(swizzle, 0)); - surf->ss7.shader_channel_select_g = swizzle_to_scs(GET_SWZ(swizzle, 1)); - surf->ss7.shader_channel_select_b = swizzle_to_scs(GET_SWZ(swizzle, 2)); - surf->ss7.shader_channel_select_a = swizzle_to_scs(GET_SWZ(swizzle, 3)); + + surf[7] = + SET_FIELD(swizzle_to_scs(GET_SWZ(swizzle, 0)), GEN7_SURFACE_SCS_R) | + SET_FIELD(swizzle_to_scs(GET_SWZ(swizzle, 1)), GEN7_SURFACE_SCS_G) | + SET_FIELD(swizzle_to_scs(GET_SWZ(swizzle, 2)), GEN7_SURFACE_SCS_B) | + SET_FIELD(swizzle_to_scs(GET_SWZ(swizzle, 3)), GEN7_SURFACE_SCS_A); } /* Emit relocation to surface contents */ drm_intel_bo_emit_reloc(brw->intel.batch.bo, - binding_table[surf_index] + - offsetof(struct gen7_surface_state, ss1), + binding_table[surf_index] + 4, intelObj->mt->region->bo, intelObj->mt->offset, I915_GEM_DOMAIN_SAMPLER, 0); @@ -408,41 +376,38 @@ gen7_create_constant_surface(struct brw_context *brw, int width, uint32_t *out_offset) { + struct intel_context *intel = &brw->intel; const GLint w = width - 1; - struct gen7_surface_state *surf; - - surf = brw_state_batch(brw, AUB_TRACE_SURFACE_STATE, - sizeof(*surf), 32, out_offset); - memset(surf, 0, sizeof(*surf)); - surf->ss0.surface_type = BRW_SURFACE_BUFFER; - surf->ss0.surface_format = BRW_SURFACEFORMAT_R32G32B32A32_FLOAT; + uint32_t *surf = brw_state_batch(brw, AUB_TRACE_SURFACE_STATE, + 8 * 4, 32, out_offset); + memset(surf, 0, 8 * 4); - surf->ss0.render_cache_read_write = 1; + surf[0] = BRW_SURFACE_BUFFER << BRW_SURFACE_TYPE_SHIFT | + BRW_SURFACEFORMAT_R32G32B32A32_FLOAT << BRW_SURFACE_FORMAT_SHIFT | + BRW_SURFACE_RC_READ_WRITE; assert(bo); - surf->ss1.base_addr = bo->offset + offset; /* reloc */ - - surf->ss2.width = w & 0x7f; /* bits 6:0 of size or width */ - surf->ss2.height = (w >> 7) & 0x1fff; /* bits 19:7 of size or width */ - surf->ss3.depth = (w >> 20) & 0x7f; /* bits 26:20 of size or width */ - surf->ss3.pitch = (16 - 1); /* stride between samples */ - gen7_set_surface_tiling(surf, I915_TILING_NONE); /* tiling now allowed */ - - if (brw->intel.is_haswell) { - surf->ss7.shader_channel_select_r = HSW_SCS_RED; - surf->ss7.shader_channel_select_g = HSW_SCS_GREEN; - surf->ss7.shader_channel_select_b = HSW_SCS_BLUE; - surf->ss7.shader_channel_select_a = HSW_SCS_ALPHA; + surf[1] = bo->offset + offset; /* reloc */ + + surf[2] = SET_FIELD(w & 0x7f, GEN7_SURFACE_WIDTH) | + SET_FIELD((w >> 7) & 0x1fff, GEN7_SURFACE_HEIGHT); + surf[3] = SET_FIELD((w >> 20) & 0x7f, BRW_SURFACE_DEPTH) | + (16 - 1); /* stride between samples */ + + if (intel->is_haswell) { + surf[7] = SET_FIELD(HSW_SCS_RED, GEN7_SURFACE_SCS_R) | + SET_FIELD(HSW_SCS_GREEN, GEN7_SURFACE_SCS_G) | + SET_FIELD(HSW_SCS_BLUE, GEN7_SURFACE_SCS_B) | + SET_FIELD(HSW_SCS_ALPHA, GEN7_SURFACE_SCS_A); } /* Emit relocation to surface contents. Section 5.1.1 of the gen4 * bspec ("Data Cache") says that the data cache does not exist as * a separate cache and is just the sampler cache. */ - drm_intel_bo_emit_reloc(brw->intel.batch.bo, - (*out_offset + - offsetof(struct gen7_surface_state, ss1)), + drm_intel_bo_emit_reloc(intel->batch.bo, + *out_offset + 4, bo, offset, I915_GEM_DOMAIN_SAMPLER, 0); @@ -469,26 +434,24 @@ gen7_update_null_renderbuffer_surface(struct brw_context *brw, unsigned unit) */ struct intel_context *intel = &brw->intel; struct gl_context *ctx = &intel->ctx; - struct gen7_surface_state *surf; /* _NEW_BUFFERS */ const struct gl_framebuffer *fb = ctx->DrawBuffer; - surf = brw_state_batch(brw, AUB_TRACE_SURFACE_STATE, - sizeof(*surf), 32, &brw->wm.surf_offset[unit]); - memset(surf, 0, sizeof(*surf)); - - surf->ss0.surface_type = BRW_SURFACE_NULL; - surf->ss0.surface_format = BRW_SURFACEFORMAT_B8G8R8A8_UNORM; - - surf->ss2.width = fb->Width - 1; - surf->ss2.height = fb->Height - 1; + uint32_t *surf = brw_state_batch(brw, AUB_TRACE_SURFACE_STATE, + 8 * 4, 32, &brw->wm.surf_offset[unit]); + memset(surf, 0, 8 * 4); - /* From the Ivy bridge PRM, Vol4 Part1 p65 (Tiled Surface: Programming Notes): - * - * If Surface Type is SURFTYPE_NULL, this field must be TRUE. + /* From the Ivybridge PRM, Volume 4, Part 1, page 65, + * Tiled Surface: Programming Notes: + * "If Surface Type is SURFTYPE_NULL, this field must be TRUE." */ - gen7_set_surface_tiling(surf, I915_TILING_Y); + surf[0] = BRW_SURFACE_NULL << BRW_SURFACE_TYPE_SHIFT | + BRW_SURFACEFORMAT_B8G8R8A8_UNORM << BRW_SURFACE_FORMAT_SHIFT | + GEN7_SURFACE_TILING_Y; + + surf[2] = SET_FIELD(fb->Width - 1, GEN7_SURFACE_WIDTH) | + SET_FIELD(fb->Height - 1, GEN7_SURFACE_HEIGHT); gen7_check_surface_setup(surf, true /* is_render_target */); } @@ -507,22 +470,17 @@ gen7_update_renderbuffer_surface(struct brw_context *brw, struct gl_context *ctx = &intel->ctx; struct intel_renderbuffer *irb = intel_renderbuffer(rb); struct intel_region *region = irb->mt->region; - struct gen7_surface_state *surf; uint32_t tile_x, tile_y; + uint32_t format; gl_format rb_format = intel_rb_format(irb); - surf = brw_state_batch(brw, AUB_TRACE_SURFACE_STATE, - sizeof(*surf), 32, &brw->wm.surf_offset[unit]); - memset(surf, 0, sizeof(*surf)); + uint32_t *surf = brw_state_batch(brw, AUB_TRACE_SURFACE_STATE, + 8 * 4, 32, &brw->wm.surf_offset[unit]); + memset(surf, 0, 8 * 4); /* Render targets can't use IMS layout */ assert(irb->mt->msaa_layout != INTEL_MSAA_LAYOUT_IMS); - if (irb->mt->align_h == 4) - surf->ss0.vertical_alignment = 1; - if (irb->mt->align_w == 8) - surf->ss0.horizontal_alignment = 1; - switch (rb_format) { case MESA_FORMAT_SARGB8: /* _NEW_BUFFERS @@ -531,27 +489,34 @@ gen7_update_renderbuffer_surface(struct brw_context *brw, * blend/update as sRGB. */ if (ctx->Color.sRGBEnabled) - surf->ss0.surface_format = brw_format_for_mesa_format(rb_format); + format = brw_format_for_mesa_format(rb_format); else - surf->ss0.surface_format = BRW_SURFACEFORMAT_B8G8R8A8_UNORM; + format = BRW_SURFACEFORMAT_B8G8R8A8_UNORM; break; default: assert(brw_render_target_supported(intel, rb)); - surf->ss0.surface_format = brw->render_target_format[rb_format]; + format = brw->render_target_format[rb_format]; if (unlikely(!brw->format_supported_as_render_target[rb_format])) { _mesa_problem(ctx, "%s: renderbuffer format %s unsupported\n", __FUNCTION__, _mesa_get_format_name(rb_format)); } - break; + break; } - surf->ss0.surface_type = BRW_SURFACE_2D; - surf->ss0.surface_array_spacing = irb->mt->array_spacing_lod0 ? - GEN7_SURFACE_ARYSPC_LOD0 : GEN7_SURFACE_ARYSPC_FULL; + surf[0] = BRW_SURFACE_2D << BRW_SURFACE_TYPE_SHIFT | + format << BRW_SURFACE_FORMAT_SHIFT | + (irb->mt->array_spacing_lod0 ? GEN7_SURFACE_ARYSPC_LOD0 + : GEN7_SURFACE_ARYSPC_FULL) | + gen7_surface_tiling_mode(region->tiling); + + if (irb->mt->align_h == 4) + surf[0] |= GEN7_SURFACE_VALIGN_4; + if (irb->mt->align_w == 8) + surf[0] |= GEN7_SURFACE_HALIGN_8; /* reloc */ - surf->ss1.base_addr = intel_renderbuffer_tile_offsets(irb, &tile_x, &tile_y); - surf->ss1.base_addr += region->bo->offset; /* reloc */ + surf[1] = intel_renderbuffer_tile_offsets(irb, &tile_x, &tile_y) + + region->bo->offset; /* reloc */ assert(brw->has_surface_tile_offset); /* Note that the low bits of these fields are missing, so @@ -559,33 +524,31 @@ gen7_update_renderbuffer_surface(struct brw_context *brw, */ assert(tile_x % 4 == 0); assert(tile_y % 2 == 0); - surf->ss5.x_offset = tile_x / 4; - surf->ss5.y_offset = tile_y / 2; + surf[5] = SET_FIELD(tile_x / 4, BRW_SURFACE_X_OFFSET) | + SET_FIELD(tile_y / 2, BRW_SURFACE_Y_OFFSET); - surf->ss2.width = rb->Width - 1; - surf->ss2.height = rb->Height - 1; - gen7_set_surface_tiling(surf, region->tiling); - surf->ss3.pitch = (region->pitch * region->cpp) - 1; + surf[2] = SET_FIELD(rb->Width - 1, GEN7_SURFACE_WIDTH) | + SET_FIELD(rb->Height - 1, GEN7_SURFACE_HEIGHT); + surf[3] = (region->pitch * region->cpp) - 1; - gen7_set_surface_msaa(surf, irb->mt->num_samples, irb->mt->msaa_layout); + surf[4] = gen7_surface_msaa_bits(irb->mt->num_samples, irb->mt->msaa_layout); if (irb->mt->msaa_layout == INTEL_MSAA_LAYOUT_CMS) { gen7_set_surface_mcs_info(brw, surf, brw->wm.surf_offset[unit], - irb->mt->mcs_mt, true /* is_render_target */); + irb->mt->mcs_mt, true /* is RT */); } if (intel->is_haswell) { - surf->ss7.shader_channel_select_r = HSW_SCS_RED; - surf->ss7.shader_channel_select_g = HSW_SCS_GREEN; - surf->ss7.shader_channel_select_b = HSW_SCS_BLUE; - surf->ss7.shader_channel_select_a = HSW_SCS_ALPHA; + surf[7] = SET_FIELD(HSW_SCS_RED, GEN7_SURFACE_SCS_R) | + SET_FIELD(HSW_SCS_GREEN, GEN7_SURFACE_SCS_G) | + SET_FIELD(HSW_SCS_BLUE, GEN7_SURFACE_SCS_B) | + SET_FIELD(HSW_SCS_ALPHA, GEN7_SURFACE_SCS_A); } drm_intel_bo_emit_reloc(brw->intel.batch.bo, - brw->wm.surf_offset[unit] + - offsetof(struct gen7_surface_state, ss1), + brw->wm.surf_offset[unit] + 4, region->bo, - surf->ss1.base_addr - region->bo->offset, + surf[1] - region->bo->offset, I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER); |