aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/intel/vulkan/anv_allocator.c66
-rw-r--r--src/intel/vulkan/anv_batch_chain.c6
-rw-r--r--src/intel/vulkan/anv_blorp.c2
-rw-r--r--src/intel/vulkan/anv_device.c15
-rw-r--r--src/intel/vulkan/anv_private.h18
-rw-r--r--src/intel/vulkan/tests/block_pool_no_free.c7
-rw-r--r--src/intel/vulkan/tests/state_pool.c4
-rw-r--r--src/intel/vulkan/tests/state_pool_free_list_only.c2
-rw-r--r--src/intel/vulkan/tests/state_pool_no_free.c4
9 files changed, 74 insertions, 50 deletions
diff --git a/src/intel/vulkan/anv_allocator.c b/src/intel/vulkan/anv_allocator.c
index 97bcb0170ba..cd300122395 100644
--- a/src/intel/vulkan/anv_allocator.c
+++ b/src/intel/vulkan/anv_allocator.c
@@ -245,20 +245,19 @@ anv_ptr_free_list_push(void **list, void *elem)
} while (old != current);
}
-static uint32_t
-anv_block_pool_grow(struct anv_block_pool *pool, struct anv_block_state *state);
+static VkResult
+anv_block_pool_expand_range(struct anv_block_pool *pool,
+ uint32_t center_bo_offset, uint32_t size);
VkResult
anv_block_pool_init(struct anv_block_pool *pool,
- struct anv_device *device, uint32_t block_size)
+ struct anv_device *device,
+ uint32_t initial_size)
{
VkResult result;
- assert(util_is_power_of_two(block_size));
-
pool->device = device;
anv_bo_init(&pool->bo, 0, 0);
- pool->block_size = block_size;
pool->free_list = ANV_FREE_LIST_EMPTY;
pool->back_free_list = ANV_FREE_LIST_EMPTY;
@@ -287,11 +286,14 @@ anv_block_pool_init(struct anv_block_pool *pool,
pool->back_state.next = 0;
pool->back_state.end = 0;
- /* Immediately grow the pool so we'll have a backing bo. */
- pool->state.end = anv_block_pool_grow(pool, &pool->state);
+ result = anv_block_pool_expand_range(pool, 0, initial_size);
+ if (result != VK_SUCCESS)
+ goto fail_mmap_cleanups;
return VK_SUCCESS;
+ fail_mmap_cleanups:
+ u_vector_finish(&pool->mmap_cleanups);
fail_fd:
close(pool->fd);
@@ -432,7 +434,8 @@ anv_block_pool_expand_range(struct anv_block_pool *pool,
* the pool and a 4K CPU page.
*/
static uint32_t
-anv_block_pool_grow(struct anv_block_pool *pool, struct anv_block_state *state)
+anv_block_pool_grow(struct anv_block_pool *pool, struct anv_block_state *state,
+ uint32_t block_size)
{
uint32_t size;
VkResult result = VK_SUCCESS;
@@ -471,7 +474,7 @@ anv_block_pool_grow(struct anv_block_pool *pool, struct anv_block_state *state)
if (old_size == 0) {
/* This is the first allocation */
- size = MAX2(32 * pool->block_size, PAGE_SIZE);
+ size = MAX2(32 * block_size, PAGE_SIZE);
} else {
size = old_size * 2;
}
@@ -500,7 +503,7 @@ anv_block_pool_grow(struct anv_block_pool *pool, struct anv_block_state *state)
center_bo_offset = ((uint64_t)size * back_used) / total_used;
/* Align down to a multiple of both the block size and page size */
- uint32_t granularity = MAX2(pool->block_size, PAGE_SIZE);
+ uint32_t granularity = MAX2(block_size, PAGE_SIZE);
assert(util_is_power_of_two(granularity));
center_bo_offset &= ~(granularity - 1);
@@ -515,7 +518,7 @@ anv_block_pool_grow(struct anv_block_pool *pool, struct anv_block_state *state)
center_bo_offset = size - pool->state.end;
}
- assert(center_bo_offset % pool->block_size == 0);
+ assert(center_bo_offset % block_size == 0);
assert(center_bo_offset % PAGE_SIZE == 0);
result = anv_block_pool_expand_range(pool, center_bo_offset, size);
@@ -544,12 +547,15 @@ done:
static uint32_t
anv_block_pool_alloc_new(struct anv_block_pool *pool,
- struct anv_block_state *pool_state)
+ struct anv_block_state *pool_state,
+ uint32_t block_size)
{
struct anv_block_state state, old, new;
+ assert(util_is_power_of_two(block_size));
+
while (1) {
- state.u64 = __sync_fetch_and_add(&pool_state->u64, pool->block_size);
+ state.u64 = __sync_fetch_and_add(&pool_state->u64, block_size);
if (state.next < state.end) {
assert(pool->map);
return state.next;
@@ -558,9 +564,8 @@ anv_block_pool_alloc_new(struct anv_block_pool *pool,
* pool_state->next acts a mutex: threads who try to allocate now will
* get block indexes above the current limit and hit futex_wait
* below. */
- new.next = state.next + pool->block_size;
- new.end = anv_block_pool_grow(pool, pool_state);
- assert(new.end >= new.next && new.end % pool->block_size == 0);
+ new.next = state.next + block_size;
+ new.end = anv_block_pool_grow(pool, pool_state, block_size);
old.u64 = __sync_lock_test_and_set(&pool_state->u64, new.u64);
if (old.next != state.next)
futex_wake(&pool_state->end, INT_MAX);
@@ -573,7 +578,8 @@ anv_block_pool_alloc_new(struct anv_block_pool *pool,
}
int32_t
-anv_block_pool_alloc(struct anv_block_pool *pool)
+anv_block_pool_alloc(struct anv_block_pool *pool,
+ uint32_t block_size)
{
int32_t offset;
@@ -584,7 +590,7 @@ anv_block_pool_alloc(struct anv_block_pool *pool)
return offset;
}
- return anv_block_pool_alloc_new(pool, &pool->state);
+ return anv_block_pool_alloc_new(pool, &pool->state, block_size);
}
/* Allocates a block out of the back of the block pool.
@@ -597,7 +603,8 @@ anv_block_pool_alloc(struct anv_block_pool *pool)
* gymnastics with the block pool's BO when doing relocations.
*/
int32_t
-anv_block_pool_alloc_back(struct anv_block_pool *pool)
+anv_block_pool_alloc_back(struct anv_block_pool *pool,
+ uint32_t block_size)
{
int32_t offset;
@@ -608,7 +615,7 @@ anv_block_pool_alloc_back(struct anv_block_pool *pool)
return offset;
}
- offset = anv_block_pool_alloc_new(pool, &pool->back_state);
+ offset = anv_block_pool_alloc_new(pool, &pool->back_state, block_size);
/* The offset we get out of anv_block_pool_alloc_new() is actually the
* number of bytes downwards from the middle to the end of the block.
@@ -616,7 +623,7 @@ anv_block_pool_alloc_back(struct anv_block_pool *pool)
* start of the block.
*/
assert(offset >= 0);
- return -(offset + pool->block_size);
+ return -(offset + block_size);
}
void
@@ -631,9 +638,12 @@ anv_block_pool_free(struct anv_block_pool *pool, int32_t offset)
void
anv_state_pool_init(struct anv_state_pool *pool,
- struct anv_block_pool *block_pool)
+ struct anv_block_pool *block_pool,
+ uint32_t block_size)
{
pool->block_pool = block_pool;
+ assert(util_is_power_of_two(block_size));
+ pool->block_size = block_size;
for (unsigned i = 0; i < ANV_STATE_BUCKETS; i++) {
pool->buckets[i].free_list = ANV_FREE_LIST_EMPTY;
pool->buckets[i].block.next = 0;
@@ -651,7 +661,8 @@ anv_state_pool_finish(struct anv_state_pool *pool)
static uint32_t
anv_fixed_size_state_pool_alloc_new(struct anv_fixed_size_state_pool *pool,
struct anv_block_pool *block_pool,
- uint32_t state_size)
+ uint32_t state_size,
+ uint32_t block_size)
{
struct anv_block_state block, old, new;
uint32_t offset;
@@ -662,9 +673,9 @@ anv_fixed_size_state_pool_alloc_new(struct anv_fixed_size_state_pool *pool,
if (block.next < block.end) {
return block.next;
} else if (block.next == block.end) {
- offset = anv_block_pool_alloc(block_pool);
+ offset = anv_block_pool_alloc(block_pool, block_size);
new.next = offset + state_size;
- new.end = offset + block_pool->block_size;
+ new.end = offset + block_size;
old.u64 = __sync_lock_test_and_set(&pool->block.u64, new.u64);
if (old.next != block.next)
futex_wake(&pool->block.end, INT_MAX);
@@ -697,7 +708,8 @@ anv_state_pool_alloc_no_vg(struct anv_state_pool *pool,
state.offset = anv_fixed_size_state_pool_alloc_new(&pool->buckets[bucket],
pool->block_pool,
- state.alloc_size);
+ state.alloc_size,
+ pool->block_size);
done:
state.map = pool->block_pool->map + state.offset;
diff --git a/src/intel/vulkan/anv_batch_chain.c b/src/intel/vulkan/anv_batch_chain.c
index 0529f22b842..79dbf266d21 100644
--- a/src/intel/vulkan/anv_batch_chain.c
+++ b/src/intel/vulkan/anv_batch_chain.c
@@ -623,12 +623,13 @@ anv_cmd_buffer_alloc_binding_table(struct anv_cmd_buffer *cmd_buffer,
{
struct anv_block_pool *block_pool =
&cmd_buffer->device->surface_state_block_pool;
+ struct anv_state_pool *state_pool = &cmd_buffer->device->surface_state_pool;
int32_t *bt_block = u_vector_head(&cmd_buffer->bt_blocks);
struct anv_state state;
state.alloc_size = align_u32(entries * 4, 32);
- if (cmd_buffer->bt_next + state.alloc_size > block_pool->block_size)
+ if (cmd_buffer->bt_next + state.alloc_size > state_pool->block_size)
return (struct anv_state) { 0 };
state.offset = cmd_buffer->bt_next;
@@ -663,6 +664,7 @@ anv_cmd_buffer_new_binding_table_block(struct anv_cmd_buffer *cmd_buffer)
{
struct anv_block_pool *block_pool =
&cmd_buffer->device->surface_state_block_pool;
+ struct anv_state_pool *state_pool = &cmd_buffer->device->surface_state_pool;
int32_t *offset = u_vector_add(&cmd_buffer->bt_blocks);
if (offset == NULL) {
@@ -670,7 +672,7 @@ anv_cmd_buffer_new_binding_table_block(struct anv_cmd_buffer *cmd_buffer)
return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
}
- *offset = anv_block_pool_alloc_back(block_pool);
+ *offset = anv_block_pool_alloc_back(block_pool, state_pool->block_size);
cmd_buffer->bt_next = 0;
return VK_SUCCESS;
diff --git a/src/intel/vulkan/anv_blorp.c b/src/intel/vulkan/anv_blorp.c
index d17b73dcc7e..e3e952060af 100644
--- a/src/intel/vulkan/anv_blorp.c
+++ b/src/intel/vulkan/anv_blorp.c
@@ -686,7 +686,7 @@ void anv_CmdUpdateBuffer(
* little data at the top to build its linked list.
*/
const uint32_t max_update_size =
- cmd_buffer->device->dynamic_state_block_pool.block_size - 64;
+ cmd_buffer->device->dynamic_state_pool.block_size - 64;
assert(max_update_size < MAX_SURFACE_DIM * 4);
diff --git a/src/intel/vulkan/anv_device.c b/src/intel/vulkan/anv_device.c
index 23dff906ac2..71967b6dbd2 100644
--- a/src/intel/vulkan/anv_device.c
+++ b/src/intel/vulkan/anv_device.c
@@ -1110,28 +1110,31 @@ VkResult anv_CreateDevice(
goto fail_batch_bo_pool;
result = anv_block_pool_init(&device->dynamic_state_block_pool, device,
- 16384);
+ 16384 * 16);
if (result != VK_SUCCESS)
goto fail_bo_cache;
anv_state_pool_init(&device->dynamic_state_pool,
- &device->dynamic_state_block_pool);
+ &device->dynamic_state_block_pool,
+ 16384);
result = anv_block_pool_init(&device->instruction_block_pool, device,
- 1024 * 1024);
+ 1024 * 1024 * 16);
if (result != VK_SUCCESS)
goto fail_dynamic_state_pool;
anv_state_pool_init(&device->instruction_state_pool,
- &device->instruction_block_pool);
+ &device->instruction_block_pool,
+ 1024 * 1024);
result = anv_block_pool_init(&device->surface_state_block_pool, device,
- 4096);
+ 4096 * 16);
if (result != VK_SUCCESS)
goto fail_instruction_state_pool;
anv_state_pool_init(&device->surface_state_pool,
- &device->surface_state_block_pool);
+ &device->surface_state_block_pool,
+ 4096);
result = anv_bo_init_new(&device->workaround_bo, device, 1024);
if (result != VK_SUCCESS)
diff --git a/src/intel/vulkan/anv_private.h b/src/intel/vulkan/anv_private.h
index 7db110e5a32..81a026451a1 100644
--- a/src/intel/vulkan/anv_private.h
+++ b/src/intel/vulkan/anv_private.h
@@ -461,8 +461,6 @@ struct anv_block_pool {
*/
struct u_vector mmap_cleanups;
- uint32_t block_size;
-
union anv_free_list free_list;
struct anv_block_state state;
@@ -504,6 +502,10 @@ struct anv_fixed_size_state_pool {
struct anv_state_pool {
struct anv_block_pool *block_pool;
+
+ /* The size of blocks which will be allocated from the block pool */
+ uint32_t block_size;
+
struct anv_fixed_size_state_pool buckets[ANV_STATE_BUCKETS];
};
@@ -555,13 +557,17 @@ anv_invalidate_range(void *start, size_t size)
}
VkResult anv_block_pool_init(struct anv_block_pool *pool,
- struct anv_device *device, uint32_t block_size);
+ struct anv_device *device,
+ uint32_t initial_size);
void anv_block_pool_finish(struct anv_block_pool *pool);
-int32_t anv_block_pool_alloc(struct anv_block_pool *pool);
-int32_t anv_block_pool_alloc_back(struct anv_block_pool *pool);
+int32_t anv_block_pool_alloc(struct anv_block_pool *pool,
+ uint32_t block_size);
+int32_t anv_block_pool_alloc_back(struct anv_block_pool *pool,
+ uint32_t block_size);
void anv_block_pool_free(struct anv_block_pool *pool, int32_t offset);
void anv_state_pool_init(struct anv_state_pool *pool,
- struct anv_block_pool *block_pool);
+ struct anv_block_pool *block_pool,
+ uint32_t block_size);
void anv_state_pool_finish(struct anv_state_pool *pool);
struct anv_state anv_state_pool_alloc(struct anv_state_pool *pool,
uint32_t state_size, uint32_t alignment);
diff --git a/src/intel/vulkan/tests/block_pool_no_free.c b/src/intel/vulkan/tests/block_pool_no_free.c
index 0a61818e42c..ac2b7801f7f 100644
--- a/src/intel/vulkan/tests/block_pool_no_free.c
+++ b/src/intel/vulkan/tests/block_pool_no_free.c
@@ -25,6 +25,7 @@
#include "anv_private.h"
+#define BLOCK_SIZE 16
#define NUM_THREADS 16
#define BLOCKS_PER_THREAD 1024
#define NUM_RUNS 64
@@ -44,13 +45,13 @@ static void *alloc_blocks(void *_job)
int32_t block, *data;
for (unsigned i = 0; i < BLOCKS_PER_THREAD; i++) {
- block = anv_block_pool_alloc(job->pool);
+ block = anv_block_pool_alloc(job->pool, BLOCK_SIZE);
data = job->pool->map + block;
*data = block;
assert(block >= 0);
job->blocks[i] = block;
- block = anv_block_pool_alloc_back(job->pool);
+ block = anv_block_pool_alloc_back(job->pool, BLOCK_SIZE);
data = job->pool->map + block;
*data = block;
assert(block < 0);
@@ -114,7 +115,7 @@ static void run_test()
struct anv_block_pool pool;
pthread_mutex_init(&device.mutex, NULL);
- anv_block_pool_init(&pool, &device, 16);
+ anv_block_pool_init(&pool, &device, 4096);
for (unsigned i = 0; i < NUM_THREADS; i++) {
jobs[i].pool = &pool;
diff --git a/src/intel/vulkan/tests/state_pool.c b/src/intel/vulkan/tests/state_pool.c
index 90c9bdea514..db3f3ec08a4 100644
--- a/src/intel/vulkan/tests/state_pool.c
+++ b/src/intel/vulkan/tests/state_pool.c
@@ -44,8 +44,8 @@ int main(int argc, char **argv)
pthread_mutex_init(&device.mutex, NULL);
for (unsigned i = 0; i < NUM_RUNS; i++) {
- anv_block_pool_init(&block_pool, &device, 256);
- anv_state_pool_init(&state_pool, &block_pool);
+ anv_block_pool_init(&block_pool, &device, 4096);
+ anv_state_pool_init(&state_pool, &block_pool, 256);
/* Grab one so a zero offset is impossible */
anv_state_pool_alloc(&state_pool, 16, 16);
diff --git a/src/intel/vulkan/tests/state_pool_free_list_only.c b/src/intel/vulkan/tests/state_pool_free_list_only.c
index 868815cf933..93b71efd437 100644
--- a/src/intel/vulkan/tests/state_pool_free_list_only.c
+++ b/src/intel/vulkan/tests/state_pool_free_list_only.c
@@ -42,7 +42,7 @@ int main(int argc, char **argv)
pthread_mutex_init(&device.mutex, NULL);
anv_block_pool_init(&block_pool, &device, 4096);
- anv_state_pool_init(&state_pool, &block_pool);
+ anv_state_pool_init(&state_pool, &block_pool, 4096);
/* Grab one so a zero offset is impossible */
anv_state_pool_alloc(&state_pool, 16, 16);
diff --git a/src/intel/vulkan/tests/state_pool_no_free.c b/src/intel/vulkan/tests/state_pool_no_free.c
index 6e012e46834..c3c7c24a0a3 100644
--- a/src/intel/vulkan/tests/state_pool_no_free.c
+++ b/src/intel/vulkan/tests/state_pool_no_free.c
@@ -62,8 +62,8 @@ static void run_test()
struct anv_state_pool state_pool;
pthread_mutex_init(&device.mutex, NULL);
- anv_block_pool_init(&block_pool, &device, 64);
- anv_state_pool_init(&state_pool, &block_pool);
+ anv_block_pool_init(&block_pool, &device, 4096);
+ anv_state_pool_init(&state_pool, &block_pool, 64);
pthread_barrier_init(&barrier, NULL, NUM_THREADS);
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-07 01:26:33 +0000