panfrost: Try to evict unused BOs from the cache

The panfrost BO cache can only grow since all newly allocated BOs are
returned to the cache (unless they've been exported).

With the MADVISE ioctl that's not a big issue because the kernel can
come and reclaim this memory, but MADVISE will only be available on 5.4
kernels. This means an app can currently allocate a lot memory without
ever releasing it, leading to some situations where the OOM-killer kicks
in and kills the app (or even worse, kills another process consuming
more memory than the GL app) to get some of this memory back.

Let's try to limit the amount of BOs we keep in the cache by evicting
entries that have not been used for more than one second (if the app
stopped allocating BOs of this size, it's likely to not allocate
similar BOs in a near future).

This solution is based on the VC4/V3D implementation.

Signed-off-by: Boris Brezillon <[email protected]>
Reviewed-by: Alyssa Rosenzweig <[email protected]>

4 files changed, 61 insertions, 6 deletions

diff --git a/src/gallium/drivers/panfrost/pan_bo.c b/src/gallium/drivers/panfrost/pan_bo.c
index 1fe7e225b8a..1a945daf237 100644
--- a/src/gallium/drivers/panfrost/pan_bo.c
+++ b/src/gallium/drivers/panfrost/pan_bo.c
@@ -203,7 +203,8 @@ panfrost_bo_cache_fetch(struct panfrost_screen *screen,
         struct panfrost_bo *bo = NULL;
 
         /* Iterate the bucket looking for something suitable */
-        list_for_each_entry_safe(struct panfrost_bo, entry, bucket, link) {
+        list_for_each_entry_safe(struct panfrost_bo, entry, bucket,
+                                 bucket_link) {
                 if (entry->size < size || entry->flags != flags)
                         continue;
 
@@ -218,7 +219,8 @@ panfrost_bo_cache_fetch(struct panfrost_screen *screen,
                 int ret;
 
                 /* This one works, splice it out of the cache */
-                list_del(&entry->link);
+                list_del(&entry->bucket_link);
+                list_del(&entry->lru_link);
 
                 ret = drmIoctl(screen->fd, DRM_IOCTL_PANFROST_MADVISE, &madv);
                 if (!ret && !madv.retained) {
@@ -234,6 +236,31 @@ panfrost_bo_cache_fetch(struct panfrost_screen *screen,
         return bo;
 }
 
+static void
+panfrost_bo_cache_evict_stale_bos(struct panfrost_screen *screen)
+{
+        struct timespec time;
+
+        clock_gettime(CLOCK_MONOTONIC, &time);
+        list_for_each_entry_safe(struct panfrost_bo, entry,
+                                 &screen->bo_cache.lru, lru_link) {
+                /* We want all entries that have been used more than 1 sec
+                 * ago to be dropped, others can be kept.
+                 * Note the <= 2 check and not <= 1. It's here to account for
+                 * the fact that we're only testing ->tv_sec, not ->tv_nsec.
+                 * That means we might keep entries that are between 1 and 2
+                 * seconds old, but we don't really care, as long as unused BOs
+                 * are dropped at some point.
+                 */
+                if (time.tv_sec - entry->last_used <= 2)
+                        break;
+
+                list_del(&entry->bucket_link);
+                list_del(&entry->lru_link);
+                panfrost_bo_free(entry);
+        }
+}
+
 /* Tries to add a BO to the cache. Returns if it was
  * successful */
 
@@ -248,6 +275,7 @@ panfrost_bo_cache_put(struct panfrost_bo *bo)
         pthread_mutex_lock(&screen->bo_cache.lock);
         struct list_head *bucket = pan_bucket(screen, bo->size);
         struct drm_panfrost_madvise madv;
+        struct timespec time;
 
         madv.handle = bo->gem_handle;
         madv.madv = PANFROST_MADV_DONTNEED;
@@ -256,7 +284,17 @@ panfrost_bo_cache_put(struct panfrost_bo *bo)
         drmIoctl(screen->fd, DRM_IOCTL_PANFROST_MADVISE, &madv);
 
         /* Add us to the bucket */
-        list_addtail(&bo->link, bucket);
+        list_addtail(&bo->bucket_link, bucket);
+
+        /* Add us to the LRU list and update the last_used field. */
+        list_addtail(&bo->lru_link, &screen->bo_cache.lru);
+        clock_gettime(CLOCK_MONOTONIC, &time);
+        bo->last_used = time.tv_sec;
+
+        /* Let's do some cleanup in the BO cache while we hold the
+         * lock.
+         */
+        panfrost_bo_cache_evict_stale_bos(screen);
         pthread_mutex_unlock(&screen->bo_cache.lock);
 
         return true;
@@ -276,8 +314,10 @@ panfrost_bo_cache_evict_all(
         for (unsigned i = 0; i < ARRAY_SIZE(screen->bo_cache.buckets); ++i) {
                 struct list_head *bucket = &screen->bo_cache.buckets[i];
 
-                list_for_each_entry_safe(struct panfrost_bo, entry, bucket, link) {
-                        list_del(&entry->link);
+                list_for_each_entry_safe(struct panfrost_bo, entry, bucket,
+                                         bucket_link) {
+                        list_del(&entry->bucket_link);
+                        list_del(&entry->lru_link);
                         panfrost_bo_free(entry);
                 }
         }
diff --git a/src/gallium/drivers/panfrost/pan_bo.h b/src/gallium/drivers/panfrost/pan_bo.h
index 78e9b7526e0..414c356b95c 100644
--- a/src/gallium/drivers/panfrost/pan_bo.h
+++ b/src/gallium/drivers/panfrost/pan_bo.h
@@ -82,7 +82,15 @@ struct panfrost_screen;
 
 struct panfrost_bo {
         /* Must be first for casting */
-        struct list_head link;
+        struct list_head bucket_link;
+
+        /* Used to link the BO to the BO cache LRU list. */
+        struct list_head lru_link;
+
+        /* Store the time this BO was use last, so the BO cache logic can evict
+         * stale BOs.
+         */
+        time_t last_used;
 
         struct pipe_reference reference;
 
diff --git a/src/gallium/drivers/panfrost/pan_screen.c b/src/gallium/drivers/panfrost/pan_screen.c
index 9e98cf3375c..c9c50e4ef18 100644
--- a/src/gallium/drivers/panfrost/pan_screen.c
+++ b/src/gallium/drivers/panfrost/pan_screen.c
@@ -755,6 +755,7 @@ panfrost_create_screen(int fd, struct renderonly *ro)
                                               panfrost_active_bos_cmp);
 
         pthread_mutex_init(&screen->bo_cache.lock, NULL);
+        list_inithead(&screen->bo_cache.lru);
         for (unsigned i = 0; i < ARRAY_SIZE(screen->bo_cache.buckets); ++i)
                 list_inithead(&screen->bo_cache.buckets[i]);
 
diff --git a/src/gallium/drivers/panfrost/pan_screen.h b/src/gallium/drivers/panfrost/pan_screen.h
index dde745fb0ed..c864629a372 100644
--- a/src/gallium/drivers/panfrost/pan_screen.h
+++ b/src/gallium/drivers/panfrost/pan_screen.h
@@ -92,6 +92,12 @@ struct panfrost_screen {
         struct {
                 pthread_mutex_t lock;
 
+                /* List containing all cached BOs sorted in LRU (Least
+                 * Recently Used) order. This allows us to quickly evict BOs
+                 * that are more than 1 second old.
+                 */
+                struct list_head lru;
+
                 /* The BO cache is a set of buckets with power-of-two sizes
                  * ranging from 2^12 (4096, the page size) to
                  * 2^(12 + MAX_BO_CACHE_BUCKETS).