diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/intel/vulkan/Makefile.am | 2 | ||||
| -rw-r--r-- | src/intel/vulkan/anv_meta_blit.c | 612 | ||||
| -rw-r--r-- | src/intel/vulkan/anv_meta_blit2d.c | 213 | ||||
| -rw-r--r-- | src/intel/vulkan/anv_meta_copy.c | 441 |
4 files changed, 662 insertions, 606 deletions
diff --git a/src/intel/vulkan/Makefile.am b/src/intel/vulkan/Makefile.am index 272db40d10b..f20cd41fbba 100644 --- a/src/intel/vulkan/Makefile.am +++ b/src/intel/vulkan/Makefile.am @@ -83,7 +83,9 @@ VULKAN_SOURCES = \ anv_intel.c \ anv_meta.c \ anv_meta_blit.c \ + anv_meta_blit2d.c \ anv_meta_clear.c \ + anv_meta_copy.c \ anv_meta_resolve.c \ anv_nir_apply_dynamic_offsets.c \ anv_nir_apply_pipeline_layout.c \ diff --git a/src/intel/vulkan/anv_meta_blit.c b/src/intel/vulkan/anv_meta_blit.c index 57833bf66ac..7bddc6b2d42 100644 --- a/src/intel/vulkan/anv_meta_blit.c +++ b/src/intel/vulkan/anv_meta_blit.c @@ -120,47 +120,6 @@ meta_prepare_blit(struct anv_cmd_buffer *cmd_buffer, } void -anv_meta_begin_blit2d(struct anv_cmd_buffer *cmd_buffer, - struct anv_meta_saved_state *save) -{ - meta_prepare_blit(cmd_buffer, save); -} - - -/* Returns the user-provided VkBufferImageCopy::imageOffset in units of - * elements rather than texels. One element equals one texel or one block - * if Image is uncompressed or compressed, respectively. - */ -static struct VkOffset3D -meta_region_offset_el(const struct anv_image * image, - const struct VkOffset3D * offset) -{ - const struct isl_format_layout * isl_layout = image->format->isl_layout; - return (VkOffset3D) { - .x = offset->x / isl_layout->bw, - .y = offset->y / isl_layout->bh, - .z = offset->z / isl_layout->bd, - }; -} - -/* Returns the user-provided VkBufferImageCopy::imageExtent in units of - * elements rather than texels. One element equals one texel or one block - * if Image is uncompressed or compressed, respectively. - */ -static struct VkExtent3D -meta_region_extent_el(const VkFormat format, - const struct VkExtent3D * extent) -{ - const struct isl_format_layout * isl_layout = - anv_format_for_vk_format(format)->isl_layout; - return (VkExtent3D) { - .width = DIV_ROUND_UP(extent->width , isl_layout->bw), - .height = DIV_ROUND_UP(extent->height, isl_layout->bh), - .depth = DIV_ROUND_UP(extent->depth , isl_layout->bd), - }; -} - -void meta_emit_blit(struct anv_cmd_buffer *cmd_buffer, struct anv_image *src_image, struct anv_image_view *src_iview, @@ -194,8 +153,10 @@ meta_emit_blit(struct anv_cmd_buffer *cmd_buffer, dest_offset.y + dest_extent.height, }, .tex_coord = { - (float)(src_offset.x + src_extent.width) / (float)src_iview->extent.width, - (float)(src_offset.y + src_extent.height) / (float)src_iview->extent.height, + (float)(src_offset.x + src_extent.width) + / (float)src_iview->extent.width, + (float)(src_offset.y + src_extent.height) + / (float)src_iview->extent.height, (float)src_offset.z / (float)src_iview->extent.depth, }, }; @@ -207,7 +168,8 @@ meta_emit_blit(struct anv_cmd_buffer *cmd_buffer, }, .tex_coord = { (float)src_offset.x / (float)src_iview->extent.width, - (float)(src_offset.y + src_extent.height) / (float)src_iview->extent.height, + (float)(src_offset.y + src_extent.height) / + (float)src_iview->extent.height, (float)src_offset.z / (float)src_iview->extent.depth, }, }; @@ -380,444 +342,6 @@ meta_finish_blit(struct anv_cmd_buffer *cmd_buffer, anv_meta_restore(saved_state, cmd_buffer); } -void -anv_meta_end_blit2d(struct anv_cmd_buffer *cmd_buffer, - struct anv_meta_saved_state *save) -{ - meta_finish_blit(cmd_buffer, save); -} - -static VkFormat -vk_format_for_size(int bs) -{ - /* The choice of UNORM and UINT formats is very intentional here. Most of - * the time, we want to use a UINT format to avoid any rounding error in - * the blit. For stencil blits, R8_UINT is required by the hardware. - * (It's the only format allowed in conjunction with W-tiling.) Also we - * intentionally use the 4-channel formats whenever we can. This is so - * that, when we do a RGB <-> RGBX copy, the two formats will line up even - * though one of them is 3/4 the size of the other. The choice of UNORM - * vs. UINT is also very intentional because Haswell doesn't handle 8 or - * 16-bit RGB UINT formats at all so we have to use UNORM there. - * Fortunately, the only time we should ever use two different formats in - * the table below is for RGB -> RGBA blits and so we will never have any - * UNORM/UINT mismatch. - */ - switch (bs) { - case 1: return VK_FORMAT_R8_UINT; - case 2: return VK_FORMAT_R8G8_UINT; - case 3: return VK_FORMAT_R8G8B8_UNORM; - case 4: return VK_FORMAT_R8G8B8A8_UNORM; - case 6: return VK_FORMAT_R16G16B16_UNORM; - case 8: return VK_FORMAT_R16G16B16A16_UNORM; - case 12: return VK_FORMAT_R32G32B32_UINT; - case 16: return VK_FORMAT_R32G32B32A32_UINT; - default: - unreachable("Invalid format block size"); - } -} - -static struct anv_meta_blit2d_surf -blit_surf_for_image(const struct anv_image* image, - const struct isl_surf *img_isl_surf) -{ - return (struct anv_meta_blit2d_surf) { - .bo = image->bo, - .tiling = img_isl_surf->tiling, - .base_offset = image->offset, - .bs = isl_format_get_layout(img_isl_surf->format)->bs, - .pitch = isl_surf_get_row_pitch(img_isl_surf), - }; -} - -void -anv_meta_blit2d(struct anv_cmd_buffer *cmd_buffer, - struct anv_meta_blit2d_surf *src, - struct anv_meta_blit2d_surf *dst, - unsigned num_rects, - struct anv_meta_blit2d_rect *rects) -{ - VkDevice vk_device = anv_device_to_handle(cmd_buffer->device); - VkFormat src_format = vk_format_for_size(src->bs); - VkFormat dst_format = vk_format_for_size(dst->bs); - VkImageUsageFlags src_usage = VK_IMAGE_USAGE_SAMPLED_BIT; - VkImageUsageFlags dst_usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; - - for (unsigned r = 0; r < num_rects; ++r) { - - /* Create VkImages */ - VkImageCreateInfo image_info = { - .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, - .imageType = VK_IMAGE_TYPE_2D, - .format = 0, /* TEMPLATE */ - .extent = { - .width = 0, /* TEMPLATE */ - .height = 0, /* TEMPLATE */ - .depth = 1, - }, - .mipLevels = 1, - .arrayLayers = 1, - .samples = 1, - .tiling = 0, /* TEMPLATE */ - .usage = 0, /* TEMPLATE */ - }; - struct anv_image_create_info anv_image_info = { - .vk_info = &image_info, - .isl_tiling_flags = 0, /* TEMPLATE */ - }; - - /* The image height is the rect height + src/dst y-offset from the - * tile-aligned base address. - */ - struct isl_tile_info tile_info; - - anv_image_info.isl_tiling_flags = 1 << src->tiling; - image_info.tiling = anv_image_info.isl_tiling_flags == ISL_TILING_LINEAR_BIT ? - VK_IMAGE_TILING_LINEAR : VK_IMAGE_TILING_OPTIMAL; - image_info.usage = src_usage; - image_info.format = src_format, - isl_tiling_get_info(&cmd_buffer->device->isl_dev, src->tiling, src->bs, &tile_info); - image_info.extent.height = rects[r].height + - rects[r].src_y % tile_info.height; - image_info.extent.width = src->pitch / src->bs; - VkImage src_image; - anv_image_create(vk_device, &anv_image_info, - &cmd_buffer->pool->alloc, &src_image); - - anv_image_info.isl_tiling_flags = 1 << dst->tiling; - image_info.tiling = anv_image_info.isl_tiling_flags == ISL_TILING_LINEAR_BIT ? - VK_IMAGE_TILING_LINEAR : VK_IMAGE_TILING_OPTIMAL; - image_info.usage = dst_usage; - image_info.format = dst_format, - isl_tiling_get_info(&cmd_buffer->device->isl_dev, dst->tiling, dst->bs, &tile_info); - image_info.extent.height = rects[r].height + - rects[r].dst_y % tile_info.height; - image_info.extent.width = dst->pitch / dst->bs; - VkImage dst_image; - anv_image_create(vk_device, &anv_image_info, - &cmd_buffer->pool->alloc, &dst_image); - - /* We could use a vk call to bind memory, but that would require - * creating a dummy memory object etc. so there's really no point. - */ - anv_image_from_handle(src_image)->bo = src->bo; - anv_image_from_handle(src_image)->offset = src->base_offset; - anv_image_from_handle(dst_image)->bo = dst->bo; - anv_image_from_handle(dst_image)->offset = dst->base_offset; - - /* Create VkImageViews */ - VkImageViewCreateInfo iview_info = { - .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, - .image = 0, /* TEMPLATE */ - .viewType = VK_IMAGE_VIEW_TYPE_2D, - .format = 0, /* TEMPLATE */ - .subresourceRange = { - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, - .baseMipLevel = 0, - .levelCount = 1, - .baseArrayLayer = 0, - .layerCount = 1 - }, - }; - uint32_t img_o = 0; - - iview_info.image = src_image; - iview_info.format = src_format; - VkOffset3D src_offset_el = {0}; - isl_surf_get_image_intratile_offset_el_xy(&cmd_buffer->device->isl_dev, - &anv_image_from_handle(src_image)-> - color_surface.isl, - rects[r].src_x, - rects[r].src_y, - &img_o, - (uint32_t*)&src_offset_el.x, - (uint32_t*)&src_offset_el.y); - - struct anv_image_view src_iview; - anv_image_view_init(&src_iview, cmd_buffer->device, - &iview_info, cmd_buffer, img_o, src_usage); - - iview_info.image = dst_image; - iview_info.format = dst_format; - VkOffset3D dst_offset_el = {0}; - isl_surf_get_image_intratile_offset_el_xy(&cmd_buffer->device->isl_dev, - &anv_image_from_handle(dst_image)-> - color_surface.isl, - rects[r].dst_x, - rects[r].dst_y, - &img_o, - (uint32_t*)&dst_offset_el.x, - (uint32_t*)&dst_offset_el.y); - struct anv_image_view dst_iview; - anv_image_view_init(&dst_iview, cmd_buffer->device, - &iview_info, cmd_buffer, img_o, dst_usage); - - /* Perform blit */ - meta_emit_blit(cmd_buffer, - anv_image_from_handle(src_image), - &src_iview, - src_offset_el, - (VkExtent3D){rects[r].width, rects[r].height, 1}, - anv_image_from_handle(dst_image), - &dst_iview, - dst_offset_el, - (VkExtent3D){rects[r].width, rects[r].height, 1}, - VK_FILTER_NEAREST); - - anv_DestroyImage(vk_device, src_image, &cmd_buffer->pool->alloc); - anv_DestroyImage(vk_device, dst_image, &cmd_buffer->pool->alloc); - } -} - -static void -do_buffer_copy(struct anv_cmd_buffer *cmd_buffer, - struct anv_bo *src, uint64_t src_offset, - struct anv_bo *dest, uint64_t dest_offset, - int width, int height, int bs) -{ - struct anv_meta_blit2d_surf b_src = { - .bo = src, - .tiling = ISL_TILING_LINEAR, - .base_offset = src_offset, - .bs = bs, - .pitch = width * bs, - }; - struct anv_meta_blit2d_surf b_dst = { - .bo = dest, - .tiling = ISL_TILING_LINEAR, - .base_offset = dest_offset, - .bs = bs, - .pitch = width * bs, - }; - struct anv_meta_blit2d_rect rect = { - .width = width, - .height = height, - }; - anv_meta_blit2d(cmd_buffer, - &b_src, - &b_dst, - 1, - &rect); -} - -void anv_CmdCopyBuffer( - VkCommandBuffer commandBuffer, - VkBuffer srcBuffer, - VkBuffer destBuffer, - uint32_t regionCount, - const VkBufferCopy* pRegions) -{ - ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); - ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer); - ANV_FROM_HANDLE(anv_buffer, dest_buffer, destBuffer); - - struct anv_meta_saved_state saved_state; - - anv_meta_begin_blit2d(cmd_buffer, &saved_state); - - for (unsigned r = 0; r < regionCount; r++) { - uint64_t src_offset = src_buffer->offset + pRegions[r].srcOffset; - uint64_t dest_offset = dest_buffer->offset + pRegions[r].dstOffset; - uint64_t copy_size = pRegions[r].size; - - /* First, we compute the biggest format that can be used with the - * given offsets and size. - */ - int bs = 16; - - int fs = ffs(src_offset) - 1; - if (fs != -1) - bs = MIN2(bs, 1 << fs); - assert(src_offset % bs == 0); - - fs = ffs(dest_offset) - 1; - if (fs != -1) - bs = MIN2(bs, 1 << fs); - assert(dest_offset % bs == 0); - - fs = ffs(pRegions[r].size) - 1; - if (fs != -1) - bs = MIN2(bs, 1 << fs); - assert(pRegions[r].size % bs == 0); - - /* This is maximum possible width/height our HW can handle */ - uint64_t max_surface_dim = 1 << 14; - - /* First, we make a bunch of max-sized copies */ - uint64_t max_copy_size = max_surface_dim * max_surface_dim * bs; - while (copy_size >= max_copy_size) { - do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset, - dest_buffer->bo, dest_offset, - max_surface_dim, max_surface_dim, bs); - copy_size -= max_copy_size; - src_offset += max_copy_size; - dest_offset += max_copy_size; - } - - uint64_t height = copy_size / (max_surface_dim * bs); - assert(height < max_surface_dim); - if (height != 0) { - uint64_t rect_copy_size = height * max_surface_dim * bs; - do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset, - dest_buffer->bo, dest_offset, - max_surface_dim, height, bs); - copy_size -= rect_copy_size; - src_offset += rect_copy_size; - dest_offset += rect_copy_size; - } - - if (copy_size != 0) { - do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset, - dest_buffer->bo, dest_offset, - copy_size / bs, 1, bs); - } - } - - anv_meta_end_blit2d(cmd_buffer, &saved_state); -} - -void anv_CmdUpdateBuffer( - VkCommandBuffer commandBuffer, - VkBuffer dstBuffer, - VkDeviceSize dstOffset, - VkDeviceSize dataSize, - const uint32_t* pData) -{ - ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); - ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer); - struct anv_meta_saved_state saved_state; - - anv_meta_begin_blit2d(cmd_buffer, &saved_state); - - /* We can't quite grab a full block because the state stream needs a - * little data at the top to build its linked list. - */ - const uint32_t max_update_size = - cmd_buffer->device->dynamic_state_block_pool.block_size - 64; - - assert(max_update_size < (1 << 14) * 4); - - while (dataSize) { - const uint32_t copy_size = MIN2(dataSize, max_update_size); - - struct anv_state tmp_data = - anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64); - - memcpy(tmp_data.map, pData, copy_size); - - int bs; - if ((copy_size & 15) == 0 && (dstOffset & 15) == 0) { - bs = 16; - } else if ((copy_size & 7) == 0 && (dstOffset & 7) == 0) { - bs = 8; - } else { - assert((copy_size & 3) == 0 && (dstOffset & 3) == 0); - bs = 4; - } - - do_buffer_copy(cmd_buffer, - &cmd_buffer->device->dynamic_state_block_pool.bo, - tmp_data.offset, - dst_buffer->bo, dst_buffer->offset + dstOffset, - copy_size / bs, 1, bs); - - dataSize -= copy_size; - dstOffset += copy_size; - pData = (void *)pData + copy_size; - } - - anv_meta_end_blit2d(cmd_buffer, &saved_state); -} - -void anv_CmdCopyImage( - VkCommandBuffer commandBuffer, - VkImage srcImage, - VkImageLayout srcImageLayout, - VkImage destImage, - VkImageLayout destImageLayout, - uint32_t regionCount, - const VkImageCopy* pRegions) -{ - ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); - ANV_FROM_HANDLE(anv_image, src_image, srcImage); - ANV_FROM_HANDLE(anv_image, dest_image, destImage); - struct anv_meta_saved_state saved_state; - - /* From the Vulkan 1.0 spec: - * - * vkCmdCopyImage can be used to copy image data between multisample - * images, but both images must have the same number of samples. - */ - assert(src_image->samples == dest_image->samples); - - anv_meta_begin_blit2d(cmd_buffer, &saved_state); - - for (unsigned r = 0; r < regionCount; r++) { - assert(pRegions[r].srcSubresource.aspectMask == - pRegions[r].dstSubresource.aspectMask); - - VkImageAspectFlags aspect = pRegions[r].srcSubresource.aspectMask; - - /* Create blit surfaces */ - struct isl_surf *src_isl_surf = - &anv_image_get_surface_for_aspect_mask(src_image, aspect)->isl; - struct isl_surf *dst_isl_surf = - &anv_image_get_surface_for_aspect_mask(dest_image, aspect)->isl; - struct anv_meta_blit2d_surf b_src = blit_surf_for_image(src_image, src_isl_surf); - struct anv_meta_blit2d_surf b_dst = blit_surf_for_image(dest_image, dst_isl_surf); - - /* Start creating blit rect */ - const VkOffset3D dst_offset_el = meta_region_offset_el(dest_image, &pRegions[r].dstOffset); - const VkOffset3D src_offset_el = meta_region_offset_el(src_image, &pRegions[r].srcOffset); - const VkExtent3D img_extent_el = meta_region_extent_el(src_image->vk_format, - &pRegions[r].extent); - struct anv_meta_blit2d_rect rect = { - .width = img_extent_el.width, - .height = img_extent_el.height, - }; - - /* Loop through each 3D or array slice */ - unsigned num_slices_3d = pRegions[r].extent.depth; - unsigned num_slices_array = pRegions[r].dstSubresource.layerCount; - unsigned slice_3d = 0; - unsigned slice_array = 0; - while (slice_3d < num_slices_3d && slice_array < num_slices_array) { - - /* Finish creating blit rect */ - isl_surf_get_image_offset_el(dst_isl_surf, - pRegions[r].dstSubresource.mipLevel, - pRegions[r].dstSubresource.baseArrayLayer + slice_array, - pRegions[r].dstOffset.z + slice_3d, - &rect.dst_x, - &rect.dst_y); - isl_surf_get_image_offset_el(src_isl_surf, - pRegions[r].srcSubresource.mipLevel, - pRegions[r].srcSubresource.baseArrayLayer + slice_array, - pRegions[r].srcOffset.z + slice_3d, - &rect.src_x, - &rect.src_y); - rect.dst_x += dst_offset_el.x; - rect.dst_y += dst_offset_el.y; - rect.src_x += src_offset_el.x; - rect.src_y += src_offset_el.y; - - /* Perform Blit */ - anv_meta_blit2d(cmd_buffer, - &b_src, - &b_dst, - 1, - &rect); - - if (dest_image->type == VK_IMAGE_TYPE_3D) - slice_3d++; - else - slice_array++; - } - } - - anv_meta_end_blit2d(cmd_buffer, &saved_state); -} - void anv_CmdBlitImage( VkCommandBuffer commandBuffer, VkImage srcImage, @@ -925,130 +449,6 @@ void anv_CmdBlitImage( meta_finish_blit(cmd_buffer, &saved_state); } -static void -meta_copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer, - struct anv_buffer* buffer, - struct anv_image* image, - uint32_t regionCount, - const VkBufferImageCopy* pRegions, - bool forward) -{ - struct anv_meta_saved_state saved_state; - - /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to - * VK_SAMPLE_COUNT_1_BIT." - */ - assert(image->samples == 1); - - anv_meta_begin_blit2d(cmd_buffer, &saved_state); - - for (unsigned r = 0; r < regionCount; r++) { - - /* Start creating blit rect */ - const VkOffset3D img_offset_el = meta_region_offset_el(image, &pRegions[r].imageOffset); - const VkExtent3D bufferExtent = { - .width = pRegions[r].bufferRowLength, - .height = pRegions[r].bufferImageHeight, - }; - const VkExtent3D buf_extent_el = meta_region_extent_el(image->vk_format, &bufferExtent); - const VkExtent3D img_extent_el = meta_region_extent_el(image->vk_format, - &pRegions[r].imageExtent); - struct anv_meta_blit2d_rect rect = { - .width = MAX2(buf_extent_el.width, img_extent_el.width), - .height = MAX2(buf_extent_el.height, img_extent_el.height), - }; - - /* Create blit surfaces */ - VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask; - const struct isl_surf *img_isl_surf = - &anv_image_get_surface_for_aspect_mask(image, aspect)->isl; - struct anv_meta_blit2d_surf img_bsurf = blit_surf_for_image(image, img_isl_surf); - struct anv_meta_blit2d_surf buf_bsurf = { - .bo = buffer->bo, - .tiling = ISL_TILING_LINEAR, - .base_offset = buffer->offset + pRegions[r].bufferOffset, - .bs = forward ? image->format->isl_layout->bs : img_bsurf.bs, - .pitch = rect.width * buf_bsurf.bs, - }; - - /* Set direction-dependent variables */ - struct anv_meta_blit2d_surf *dst_bsurf = forward ? &img_bsurf : &buf_bsurf; - struct anv_meta_blit2d_surf *src_bsurf = forward ? &buf_bsurf : &img_bsurf; - uint32_t *x_offset = forward ? &rect.dst_x : &rect.src_x; - uint32_t *y_offset = forward ? &rect.dst_y : &rect.src_y; - - /* Loop through each 3D or array slice */ - unsigned num_slices_3d = pRegions[r].imageExtent.depth; - unsigned num_slices_array = pRegions[r].imageSubresource.layerCount; - unsigned slice_3d = 0; - unsigned slice_array = 0; - while (slice_3d < num_slices_3d && slice_array < num_slices_array) { - - /* Finish creating blit rect */ - isl_surf_get_image_offset_el(img_isl_surf, - pRegions[r].imageSubresource.mipLevel, - pRegions[r].imageSubresource.baseArrayLayer + slice_array, - pRegions[r].imageOffset.z + slice_3d, - x_offset, - y_offset); - *x_offset += img_offset_el.x; - *y_offset += img_offset_el.y; - - /* Perform Blit */ - anv_meta_blit2d(cmd_buffer, - src_bsurf, - dst_bsurf, - 1, - &rect); - - /* Once we've done the blit, all of the actual information about - * the image is embedded in the command buffer so we can just - * increment the offset directly in the image effectively - * re-binding it to different backing memory. - */ - buf_bsurf.base_offset += rect.width * rect.height * buf_bsurf.bs; - - if (image->type == VK_IMAGE_TYPE_3D) - slice_3d++; - else - slice_array++; - } - } - anv_meta_end_blit2d(cmd_buffer, &saved_state); -} - -void anv_CmdCopyBufferToImage( - VkCommandBuffer commandBuffer, - VkBuffer srcBuffer, - VkImage destImage, - VkImageLayout destImageLayout, - uint32_t regionCount, - const VkBufferImageCopy* pRegions) -{ - ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); - ANV_FROM_HANDLE(anv_image, dest_image, destImage); - ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer); - - meta_copy_buffer_to_image(cmd_buffer, src_buffer, dest_image, - regionCount, pRegions, true); -} - -void anv_CmdCopyImageToBuffer( - VkCommandBuffer commandBuffer, - VkImage srcImage, - VkImageLayout srcImageLayout, - VkBuffer destBuffer, - uint32_t regionCount, - const VkBufferImageCopy* pRegions) -{ - ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); - ANV_FROM_HANDLE(anv_image, src_image, srcImage); - ANV_FROM_HANDLE(anv_buffer, dst_buffer, destBuffer); - - meta_copy_buffer_to_image(cmd_buffer, dst_buffer, src_image, - regionCount, pRegions, false); -} - void anv_device_finish_meta_blit_state(struct anv_device *device) { diff --git a/src/intel/vulkan/anv_meta_blit2d.c b/src/intel/vulkan/anv_meta_blit2d.c new file mode 100644 index 00000000000..b165abd9b6c --- /dev/null +++ b/src/intel/vulkan/anv_meta_blit2d.c @@ -0,0 +1,213 @@ +/* + * Copyright © 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "anv_meta.h" + +static VkFormat +vk_format_for_size(int bs) +{ + /* The choice of UNORM and UINT formats is very intentional here. Most of + * the time, we want to use a UINT format to avoid any rounding error in + * the blit. For stencil blits, R8_UINT is required by the hardware. + * (It's the only format allowed in conjunction with W-tiling.) Also we + * intentionally use the 4-channel formats whenever we can. This is so + * that, when we do a RGB <-> RGBX copy, the two formats will line up even + * though one of them is 3/4 the size of the other. The choice of UNORM + * vs. UINT is also very intentional because Haswell doesn't handle 8 or + * 16-bit RGB UINT formats at all so we have to use UNORM there. + * Fortunately, the only time we should ever use two different formats in + * the table below is for RGB -> RGBA blits and so we will never have any + * UNORM/UINT mismatch. + */ + switch (bs) { + case 1: return VK_FORMAT_R8_UINT; + case 2: return VK_FORMAT_R8G8_UINT; + case 3: return VK_FORMAT_R8G8B8_UNORM; + case 4: return VK_FORMAT_R8G8B8A8_UNORM; + case 6: return VK_FORMAT_R16G16B16_UNORM; + case 8: return VK_FORMAT_R16G16B16A16_UNORM; + case 12: return VK_FORMAT_R32G32B32_UINT; + case 16: return VK_FORMAT_R32G32B32A32_UINT; + default: + unreachable("Invalid format block size"); + } +} + +void +anv_meta_end_blit2d(struct anv_cmd_buffer *cmd_buffer, + struct anv_meta_saved_state *save) +{ + anv_meta_restore(save, cmd_buffer); +} + +void +anv_meta_begin_blit2d(struct anv_cmd_buffer *cmd_buffer, + struct anv_meta_saved_state *save) +{ + anv_meta_save(save, cmd_buffer, + (1 << VK_DYNAMIC_STATE_VIEWPORT)); +} + +void +anv_meta_blit2d(struct anv_cmd_buffer *cmd_buffer, + struct anv_meta_blit2d_surf *src, + struct anv_meta_blit2d_surf *dst, + unsigned num_rects, + struct anv_meta_blit2d_rect *rects) +{ + VkDevice vk_device = anv_device_to_handle(cmd_buffer->device); + VkFormat src_format = vk_format_for_size(src->bs); + VkFormat dst_format = vk_format_for_size(dst->bs); + VkImageUsageFlags src_usage = VK_IMAGE_USAGE_SAMPLED_BIT; + VkImageUsageFlags dst_usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; + + for (unsigned r = 0; r < num_rects; ++r) { + + /* Create VkImages */ + VkImageCreateInfo image_info = { + .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, + .imageType = VK_IMAGE_TYPE_2D, + .format = 0, /* TEMPLATE */ + .extent = { + .width = 0, /* TEMPLATE */ + .height = 0, /* TEMPLATE */ + .depth = 1, + }, + .mipLevels = 1, + .arrayLayers = 1, + .samples = 1, + .tiling = 0, /* TEMPLATE */ + .usage = 0, /* TEMPLATE */ + }; + struct anv_image_create_info anv_image_info = { + .vk_info = &image_info, + .isl_tiling_flags = 0, /* TEMPLATE */ + }; + + /* The image height is the rect height + src/dst y-offset from the + * tile-aligned base address. + */ + struct isl_tile_info tile_info; + + anv_image_info.isl_tiling_flags = 1 << src->tiling; + image_info.tiling = anv_image_info.isl_tiling_flags == + ISL_TILING_LINEAR_BIT ? + VK_IMAGE_TILING_LINEAR : VK_IMAGE_TILING_OPTIMAL; + image_info.usage = src_usage; + image_info.format = src_format, + isl_tiling_get_info(&cmd_buffer->device->isl_dev, src->tiling, src->bs, + &tile_info); + image_info.extent.height = rects[r].height + + rects[r].src_y % tile_info.height; + image_info.extent.width = src->pitch / src->bs; + VkImage src_image; + anv_image_create(vk_device, &anv_image_info, + &cmd_buffer->pool->alloc, &src_image); + + anv_image_info.isl_tiling_flags = 1 << dst->tiling; + image_info.tiling = anv_image_info.isl_tiling_flags == + ISL_TILING_LINEAR_BIT ? + VK_IMAGE_TILING_LINEAR : VK_IMAGE_TILING_OPTIMAL; + image_info.usage = dst_usage; + image_info.format = dst_format, + isl_tiling_get_info(&cmd_buffer->device->isl_dev, dst->tiling, dst->bs, + &tile_info); + image_info.extent.height = rects[r].height + + rects[r].dst_y % tile_info.height; + image_info.extent.width = dst->pitch / dst->bs; + VkImage dst_image; + anv_image_create(vk_device, &anv_image_info, + &cmd_buffer->pool->alloc, &dst_image); + + /* We could use a vk call to bind memory, but that would require + * creating a dummy memory object etc. so there's really no point. + */ + anv_image_from_handle(src_image)->bo = src->bo; + anv_image_from_handle(src_image)->offset = src->base_offset; + anv_image_from_handle(dst_image)->bo = dst->bo; + anv_image_from_handle(dst_image)->offset = dst->base_offset; + + /* Create VkImageViews */ + VkImageViewCreateInfo iview_info = { + .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, + .image = 0, /* TEMPLATE */ + .viewType = VK_IMAGE_VIEW_TYPE_2D, + .format = 0, /* TEMPLATE */ + .subresourceRange = { + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = 1 + }, + }; + uint32_t img_o = 0; + + iview_info.image = src_image; + iview_info.format = src_format; + VkOffset3D src_offset_el = {0}; + isl_surf_get_image_intratile_offset_el_xy(&cmd_buffer->device->isl_dev, + &anv_image_from_handle(src_image)-> + color_surface.isl, + rects[r].src_x, + rects[r].src_y, + &img_o, + (uint32_t*)&src_offset_el.x, + (uint32_t*)&src_offset_el.y); + + struct anv_image_view src_iview; + anv_image_view_init(&src_iview, cmd_buffer->device, + &iview_info, cmd_buffer, img_o, src_usage); + + iview_info.image = dst_image; + iview_info.format = dst_format; + VkOffset3D dst_offset_el = {0}; + isl_surf_get_image_intratile_offset_el_xy(&cmd_buffer->device->isl_dev, + &anv_image_from_handle(dst_image)-> + color_surface.isl, + rects[r].dst_x, + rects[r].dst_y, + &img_o, + (uint32_t*)&dst_offset_el.x, + (uint32_t*)&dst_offset_el.y); + struct anv_image_view dst_iview; + anv_image_view_init(&dst_iview, cmd_buffer->device, + &iview_info, cmd_buffer, img_o, dst_usage); + + /* Perform blit */ + meta_emit_blit(cmd_buffer, + anv_image_from_handle(src_image), + &src_iview, + src_offset_el, + (VkExtent3D){rects[r].width, rects[r].height, 1}, + anv_image_from_handle(dst_image), + &dst_iview, + dst_offset_el, + (VkExtent3D){rects[r].width, rects[r].height, 1}, + VK_FILTER_NEAREST); + + anv_DestroyImage(vk_device, src_image, &cmd_buffer->pool->alloc); + anv_DestroyImage(vk_device, dst_image, &cmd_buffer->pool->alloc); + } +} + diff --git a/src/intel/vulkan/anv_meta_copy.c b/src/intel/vulkan/anv_meta_copy.c new file mode 100644 index 00000000000..1a2bfd6cf01 --- /dev/null +++ b/src/intel/vulkan/anv_meta_copy.c @@ -0,0 +1,441 @@ +/* + * Copyright © 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "anv_meta.h" + +/* Returns the user-provided VkBufferImageCopy::imageExtent in units of + * elements rather than texels. One element equals one texel or one block + * if Image is uncompressed or compressed, respectively. + */ +static struct VkExtent3D +meta_region_extent_el(const VkFormat format, + const struct VkExtent3D *extent) +{ + const struct isl_format_layout *isl_layout = + anv_format_for_vk_format(format)->isl_layout; + return (VkExtent3D) { + .width = DIV_ROUND_UP(extent->width , isl_layout->bw), + .height = DIV_ROUND_UP(extent->height, isl_layout->bh), + .depth = DIV_ROUND_UP(extent->depth , isl_layout->bd), + }; +} + +/* Returns the user-provided VkBufferImageCopy::imageOffset in units of + * elements rather than texels. One element equals one texel or one block + * if Image is uncompressed or compressed, respectively. + */ +static struct VkOffset3D +meta_region_offset_el(const struct anv_image *image, + const struct VkOffset3D *offset) +{ + const struct isl_format_layout *isl_layout = image->format->isl_layout; + return (VkOffset3D) { + .x = offset->x / isl_layout->bw, + .y = offset->y / isl_layout->bh, + .z = offset->z / isl_layout->bd, + }; +} + +static struct anv_meta_blit2d_surf +blit_surf_for_image(const struct anv_image* image, + const struct isl_surf *img_isl_surf) +{ + return (struct anv_meta_blit2d_surf) { + .bo = image->bo, + .tiling = img_isl_surf->tiling, + .base_offset = image->offset, + .bs = isl_format_get_layout(img_isl_surf->format)->bs, + .pitch = isl_surf_get_row_pitch(img_isl_surf), + }; +} + +static void +do_buffer_copy(struct anv_cmd_buffer *cmd_buffer, + struct anv_bo *src, uint64_t src_offset, + struct anv_bo *dest, uint64_t dest_offset, + int width, int height, int bs) +{ + struct anv_meta_blit2d_surf b_src = { + .bo = src, + .tiling = ISL_TILING_LINEAR, + .base_offset = src_offset, + .bs = bs, + .pitch = width * bs, + }; + struct anv_meta_blit2d_surf b_dst = { + .bo = dest, + .tiling = ISL_TILING_LINEAR, + .base_offset = dest_offset, + .bs = bs, + .pitch = width * bs, + }; + struct anv_meta_blit2d_rect rect = { + .width = width, + .height = height, + }; + anv_meta_blit2d(cmd_buffer, &b_src, &b_dst, 1, &rect); +} + +static void +meta_copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer, + struct anv_buffer* buffer, + struct anv_image* image, + uint32_t regionCount, + const VkBufferImageCopy* pRegions, + bool forward) +{ + struct anv_meta_saved_state saved_state; + + /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to + * VK_SAMPLE_COUNT_1_BIT." + */ + assert(image->samples == 1); + + anv_meta_begin_blit2d(cmd_buffer, &saved_state); + + for (unsigned r = 0; r < regionCount; r++) { + + /* Start creating blit rect */ + const VkOffset3D img_offset_el = + meta_region_offset_el(image, &pRegions[r].imageOffset); + const VkExtent3D bufferExtent = { + .width = pRegions[r].bufferRowLength, + .height = pRegions[r].bufferImageHeight, + }; + const VkExtent3D buf_extent_el = + meta_region_extent_el(image->vk_format, &bufferExtent); + const VkExtent3D img_extent_el = + meta_region_extent_el(image->vk_format, &pRegions[r].imageExtent); + struct anv_meta_blit2d_rect rect = { + .width = MAX2(buf_extent_el.width, img_extent_el.width), + .height = MAX2(buf_extent_el.height, img_extent_el.height), + }; + + /* Create blit surfaces */ + VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask; + const struct isl_surf *img_isl_surf = + &anv_image_get_surface_for_aspect_mask(image, aspect)->isl; + struct anv_meta_blit2d_surf img_bsurf = + blit_surf_for_image(image, img_isl_surf); + struct anv_meta_blit2d_surf buf_bsurf = { + .bo = buffer->bo, + .tiling = ISL_TILING_LINEAR, + .base_offset = buffer->offset + pRegions[r].bufferOffset, + .bs = forward ? image->format->isl_layout->bs : img_bsurf.bs, + .pitch = rect.width * buf_bsurf.bs, + }; + + /* Set direction-dependent variables */ + struct anv_meta_blit2d_surf *dst_bsurf = forward ? &img_bsurf : &buf_bsurf; + struct anv_meta_blit2d_surf *src_bsurf = forward ? &buf_bsurf : &img_bsurf; + uint32_t *x_offset = forward ? &rect.dst_x : &rect.src_x; + uint32_t *y_offset = forward ? &rect.dst_y : &rect.src_y; + + /* Loop through each 3D or array slice */ + unsigned num_slices_3d = pRegions[r].imageExtent.depth; + unsigned num_slices_array = pRegions[r].imageSubresource.layerCount; + unsigned slice_3d = 0; + unsigned slice_array = 0; + while (slice_3d < num_slices_3d && slice_array < num_slices_array) { + + /* Finish creating blit rect */ + isl_surf_get_image_offset_el(img_isl_surf, + pRegions[r].imageSubresource.mipLevel, + pRegions[r].imageSubresource.baseArrayLayer + + slice_array, + pRegions[r].imageOffset.z + slice_3d, + x_offset, + y_offset); + *x_offset += img_offset_el.x; + *y_offset += img_offset_el.y; + + /* Perform Blit */ + anv_meta_blit2d(cmd_buffer, src_bsurf, dst_bsurf, 1, &rect); + + /* Once we've done the blit, all of the actual information about + * the image is embedded in the command buffer so we can just + * increment the offset directly in the image effectively + * re-binding it to different backing memory. + */ + buf_bsurf.base_offset += rect.width * rect.height * buf_bsurf.bs; + + if (image->type == VK_IMAGE_TYPE_3D) + slice_3d++; + else + slice_array++; + } + } + anv_meta_end_blit2d(cmd_buffer, &saved_state); +} + +void anv_CmdCopyBufferToImage( + VkCommandBuffer commandBuffer, + VkBuffer srcBuffer, + VkImage destImage, + VkImageLayout destImageLayout, + uint32_t regionCount, + const VkBufferImageCopy* pRegions) +{ + ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); + ANV_FROM_HANDLE(anv_image, dest_image, destImage); + ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer); + + meta_copy_buffer_to_image(cmd_buffer, src_buffer, dest_image, + regionCount, pRegions, true); +} + +void anv_CmdCopyImageToBuffer( + VkCommandBuffer commandBuffer, + VkImage srcImage, + VkImageLayout srcImageLayout, + VkBuffer destBuffer, + uint32_t regionCount, + const VkBufferImageCopy* pRegions) +{ + ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); + ANV_FROM_HANDLE(anv_image, src_image, srcImage); + ANV_FROM_HANDLE(anv_buffer, dst_buffer, destBuffer); + + meta_copy_buffer_to_image(cmd_buffer, dst_buffer, src_image, + regionCount, pRegions, false); +} + +void anv_CmdCopyImage( + VkCommandBuffer commandBuffer, + VkImage srcImage, + VkImageLayout srcImageLayout, + VkImage destImage, + VkImageLayout destImageLayout, + uint32_t regionCount, + const VkImageCopy* pRegions) +{ + ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); + ANV_FROM_HANDLE(anv_image, src_image, srcImage); + ANV_FROM_HANDLE(anv_image, dest_image, destImage); + struct anv_meta_saved_state saved_state; + + /* From the Vulkan 1.0 spec: + * + * vkCmdCopyImage can be used to copy image data between multisample + * images, but both images must have the same number of samples. + */ + assert(src_image->samples == dest_image->samples); + + anv_meta_begin_blit2d(cmd_buffer, &saved_state); + + for (unsigned r = 0; r < regionCount; r++) { + assert(pRegions[r].srcSubresource.aspectMask == + pRegions[r].dstSubresource.aspectMask); + + VkImageAspectFlags aspect = pRegions[r].srcSubresource.aspectMask; + + /* Create blit surfaces */ + struct isl_surf *src_isl_surf = + &anv_image_get_surface_for_aspect_mask(src_image, aspect)->isl; + struct isl_surf *dst_isl_surf = + &anv_image_get_surface_for_aspect_mask(dest_image, aspect)->isl; + struct anv_meta_blit2d_surf b_src = + blit_surf_for_image(src_image, src_isl_surf); + struct anv_meta_blit2d_surf b_dst = + blit_surf_for_image(dest_image, dst_isl_surf); + + /* Start creating blit rect */ + const VkOffset3D dst_offset_el = + meta_region_offset_el(dest_image, &pRegions[r].dstOffset); + const VkOffset3D src_offset_el = + meta_region_offset_el(src_image, &pRegions[r].srcOffset); + const VkExtent3D img_extent_el = + meta_region_extent_el(src_image->vk_format, &pRegions[r].extent); + struct anv_meta_blit2d_rect rect = { + .width = img_extent_el.width, + .height = img_extent_el.height, + }; + + /* Loop through each 3D or array slice */ + unsigned num_slices_3d = pRegions[r].extent.depth; + unsigned num_slices_array = pRegions[r].dstSubresource.layerCount; + unsigned slice_3d = 0; + unsigned slice_array = 0; + while (slice_3d < num_slices_3d && slice_array < num_slices_array) { + + /* Finish creating blit rect */ + isl_surf_get_image_offset_el(dst_isl_surf, + pRegions[r].dstSubresource.mipLevel, + pRegions[r].dstSubresource.baseArrayLayer + + slice_array, + pRegions[r].dstOffset.z + slice_3d, + &rect.dst_x, + &rect.dst_y); + isl_surf_get_image_offset_el(src_isl_surf, + pRegions[r].srcSubresource.mipLevel, + pRegions[r].srcSubresource.baseArrayLayer + + slice_array, + pRegions[r].srcOffset.z + slice_3d, + &rect.src_x, + &rect.src_y); + rect.dst_x += dst_offset_el.x; + rect.dst_y += dst_offset_el.y; + rect.src_x += src_offset_el.x; + rect.src_y += src_offset_el.y; + + /* Perform Blit */ + anv_meta_blit2d(cmd_buffer, &b_src, &b_dst, 1, &rect); + + if (dest_image->type == VK_IMAGE_TYPE_3D) + slice_3d++; + else + slice_array++; + } + } + + anv_meta_end_blit2d(cmd_buffer, &saved_state); +} + +void anv_CmdCopyBuffer( + VkCommandBuffer commandBuffer, + VkBuffer srcBuffer, + VkBuffer destBuffer, + uint32_t regionCount, + const VkBufferCopy* pRegions) +{ + ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); + ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer); + ANV_FROM_HANDLE(anv_buffer, dest_buffer, destBuffer); + + struct anv_meta_saved_state saved_state; + + anv_meta_begin_blit2d(cmd_buffer, &saved_state); + + for (unsigned r = 0; r < regionCount; r++) { + uint64_t src_offset = src_buffer->offset + pRegions[r].srcOffset; + uint64_t dest_offset = dest_buffer->offset + pRegions[r].dstOffset; + uint64_t copy_size = pRegions[r].size; + + /* First, we compute the biggest format that can be used with the + * given offsets and size. + */ + int bs = 16; + + int fs = ffs(src_offset) - 1; + if (fs != -1) + bs = MIN2(bs, 1 << fs); + assert(src_offset % bs == 0); + + fs = ffs(dest_offset) - 1; + if (fs != -1) + bs = MIN2(bs, 1 << fs); + assert(dest_offset % bs == 0); + + fs = ffs(pRegions[r].size) - 1; + if (fs != -1) + bs = MIN2(bs, 1 << fs); + assert(pRegions[r].size % bs == 0); + + /* This is maximum possible width/height our HW can handle */ + uint64_t max_surface_dim = 1 << 14; + + /* First, we make a bunch of max-sized copies */ + uint64_t max_copy_size = max_surface_dim * max_surface_dim * bs; + while (copy_size >= max_copy_size) { + do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset, + dest_buffer->bo, dest_offset, + max_surface_dim, max_surface_dim, bs); + copy_size -= max_copy_size; + src_offset += max_copy_size; + dest_offset += max_copy_size; + } + + uint64_t height = copy_size / (max_surface_dim * bs); + assert(height < max_surface_dim); + if (height != 0) { + uint64_t rect_copy_size = height * max_surface_dim * bs; + do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset, + dest_buffer->bo, dest_offset, + max_surface_dim, height, bs); + copy_size -= rect_copy_size; + src_offset += rect_copy_size; + dest_offset += rect_copy_size; + } + + if (copy_size != 0) { + do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset, + dest_buffer->bo, dest_offset, + copy_size / bs, 1, bs); + } + } + + anv_meta_end_blit2d(cmd_buffer, &saved_state); +} + +void anv_CmdUpdateBuffer( + VkCommandBuffer commandBuffer, + VkBuffer dstBuffer, + VkDeviceSize dstOffset, + VkDeviceSize dataSize, + const uint32_t* pData) +{ + ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); + ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer); + struct anv_meta_saved_state saved_state; + + anv_meta_begin_blit2d(cmd_buffer, &saved_state); + + /* We can't quite grab a full block because the state stream needs a + * little data at the top to build its linked list. + */ + const uint32_t max_update_size = + cmd_buffer->device->dynamic_state_block_pool.block_size - 64; + + assert(max_update_size < (1 << 14) * 4); + + while (dataSize) { + const uint32_t copy_size = MIN2(dataSize, max_update_size); + + struct anv_state tmp_data = + anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64); + + memcpy(tmp_data.map, pData, copy_size); + + int bs; + if ((copy_size & 15) == 0 && (dstOffset & 15) == 0) { + bs = 16; + } else if ((copy_size & 7) == 0 && (dstOffset & 7) == 0) { + bs = 8; + } else { + assert((copy_size & 3) == 0 && (dstOffset & 3) == 0); + bs = 4; + } + + do_buffer_copy(cmd_buffer, + &cmd_buffer->device->dynamic_state_block_pool.bo, + tmp_data.offset, + dst_buffer->bo, dst_buffer->offset + dstOffset, + copy_size / bs, 1, bs); + + dataSize -= copy_size; + dstOffset += copy_size; + pData = (void *)pData + copy_size; + } + + anv_meta_end_blit2d(cmd_buffer, &saved_state); +} |