llvmpipe: limit the number of fragment shader variants kept around

llvmpipe can create a large number of shader variants for a single shader
(which are quite big), and they were only ever deleted if the shader itself
was deleted. This is especially apparent in things like glean
blendFunc where a new variant is created for every different subtest, chewing
up all memory.
This change limits the numbers of fragment shader variants (for all shaders)
which are kept around to a fixed number. If that would be exceeded a fixed
portion of the cached variants is deleted (since without tracking the used
variants this involves flushing we don't want to delete only one).
Always the least recently used variants (from all shaders together) are
deleted.
For now this is all per-context.
Both the number of how many variants are cached (1024) as well as how many
will be deleted at once (1/4 of the cache size) are just rough guesses and
subject to further optimization.

5 files changed, 107 insertions, 35 deletions

diff --git a/src/gallium/drivers/llvmpipe/lp_context.c b/src/gallium/drivers/llvmpipe/lp_context.c
index 9e88a6e09f4..06689c20eb8 100644
--- a/src/gallium/drivers/llvmpipe/lp_context.c
+++ b/src/gallium/drivers/llvmpipe/lp_context.c
@@ -36,6 +36,7 @@
 #include "util/u_inlines.h"
 #include "util/u_math.h"
 #include "util/u_memory.h"
+#include "util/u_simple_list.h"
 #include "lp_clear.h"
 #include "lp_context.h"
 #include "lp_flush.h"
@@ -94,6 +95,8 @@ llvmpipe_create_context( struct pipe_screen *screen, void *priv )
 
    memset(llvmpipe, 0, sizeof *llvmpipe);
 
+   make_empty_list(&llvmpipe->fs_variants_list);
+
    llvmpipe->pipe.winsys = screen->winsys;
    llvmpipe->pipe.screen = screen;
    llvmpipe->pipe.priv = priv;
diff --git a/src/gallium/drivers/llvmpipe/lp_context.h b/src/gallium/drivers/llvmpipe/lp_context.h
index cb04d4a4d54..1bdd0f79e19 100644
--- a/src/gallium/drivers/llvmpipe/lp_context.h
+++ b/src/gallium/drivers/llvmpipe/lp_context.h
@@ -38,6 +38,7 @@
 #include "lp_tex_sample.h"
 #include "lp_jit.h"
 #include "lp_setup.h"
+#include "lp_state_fs.h"
 
 
 struct llvmpipe_vbuf_render;
@@ -105,6 +106,8 @@ struct llvmpipe_context {
    unsigned tex_timestamp;
    boolean no_rast;
 
+   struct lp_fs_variant_list_item fs_variants_list;
+   unsigned nr_fs_variants;
 };
 
 
diff --git a/src/gallium/drivers/llvmpipe/lp_limits.h b/src/gallium/drivers/llvmpipe/lp_limits.h
index 4102a9df67c..d1c431475d8 100644
--- a/src/gallium/drivers/llvmpipe/lp_limits.h
+++ b/src/gallium/drivers/llvmpipe/lp_limits.h
@@ -66,5 +66,10 @@
  */
 #define LP_MAX_SCENE_SIZE (512 * 1024 * 1024)
 
+/**
+ * Max number of shader variants (for all shaders combined,
+ * per context) that will be kept around.
+ */
+#define LP_MAX_SHADER_VARIANTS 1024
 
 #endif /* LP_LIMITS_H */
diff --git a/src/gallium/drivers/llvmpipe/lp_state_fs.c b/src/gallium/drivers/llvmpipe/lp_state_fs.c
index 2619e043fdf..48c7754e41a 100644
--- a/src/gallium/drivers/llvmpipe/lp_state_fs.c
+++ b/src/gallium/drivers/llvmpipe/lp_state_fs.c
@@ -68,6 +68,7 @@
 #include "util/u_format.h"
 #include "util/u_dump.h"
 #include "util/u_string.h"
+#include "util/u_simple_list.h"
 #include "os/os_time.h"
 #include "pipe/p_shader_tokens.h"
 #include "draw/draw_context.h"
@@ -95,6 +96,7 @@
 #include "lp_setup.h"
 #include "lp_state.h"
 #include "lp_tex_sample.h"
+#include "lp_flush.h"
 
 
 #include <llvm-c/Analysis.h>
@@ -936,7 +938,10 @@ generate_variant(struct llvmpipe_context *lp,
    if(!variant)
       return NULL;
 
-   variant->no = shader->variant_no++;
+   variant->lpfs = shader;
+   variant->list_item_global.base = variant;
+   variant->list_item_local.base = variant;
+   variant->no = shader->variants_created++;
 
    memcpy(&variant->key, key, sizeof *key);
 
@@ -962,10 +967,6 @@ generate_variant(struct llvmpipe_context *lp,
          !shader->info.uses_kill
          ? TRUE : FALSE;
 
-   /* insert new variant into linked list */
-   variant->next = shader->variants;
-   shader->variants = variant;
-
    return variant;
 }
 
@@ -981,6 +982,7 @@ llvmpipe_create_fs_state(struct pipe_context *pipe,
       return NULL;
 
    shader->no = fs_no++;
+   make_empty_list(&shader->variants);
 
    /* get/save the summary info for this shader */
    tgsi_scan_shader(templ->tokens, &shader->info);
@@ -1024,14 +1026,40 @@ llvmpipe_bind_fs_state(struct pipe_context *pipe, void *fs)
    llvmpipe->dirty |= LP_NEW_FS;
 }
 
+static void
+remove_shader_variant(struct llvmpipe_context *lp,
+                      struct lp_fragment_shader_variant *variant)
+{
+   struct llvmpipe_screen *screen = llvmpipe_screen(lp->pipe.screen);
+   unsigned i;
+
+   if (gallivm_debug & GALLIVM_DEBUG_IR) {
+      debug_printf("llvmpipe: del fs #%u var #%u v created #%u v cached #%u v total cached #%u\n",
+                    variant->lpfs->no, variant->no, variant->lpfs->variants_created,
+                    variant->lpfs->variants_cached, lp->nr_fs_variants);
+   }
+   for (i = 0; i < Elements(variant->function); i++) {
+      if (variant->function[i]) {
+         if (variant->jit_function[i])
+            LLVMFreeMachineCodeForFunction(screen->engine,
+                                           variant->function[i]);
+         LLVMDeleteFunction(variant->function[i]);
+      }
+   }
+   remove_from_list(&variant->list_item_local);
+   variant->lpfs->variants_cached--;
+   remove_from_list(&variant->list_item_global);
+   lp->nr_fs_variants--;
+   FREE(variant);
+}
 
 static void
 llvmpipe_delete_fs_state(struct pipe_context *pipe, void *fs)
 {
    struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
-   struct llvmpipe_screen *screen = llvmpipe_screen(pipe->screen);
+   struct pipe_fence_handle *fence = NULL;
    struct lp_fragment_shader *shader = fs;
-   struct lp_fragment_shader_variant *variant;
+   struct lp_fs_variant_list_item *li;
 
    assert(fs != llvmpipe->fs);
    (void) llvmpipe;
@@ -1039,29 +1067,23 @@ llvmpipe_delete_fs_state(struct pipe_context *pipe, void *fs)
    /*
     * XXX: we need to flush the context until we have some sort of reference
     * counting in fragment shaders as they may still be binned
+    * Flushing alone might not sufficient we need to wait on it too.
     */
-   draw_flush(llvmpipe->draw);
-   lp_setup_flush(llvmpipe->setup, 0);
-
-   variant = shader->variants;
-   while(variant) {
-      struct lp_fragment_shader_variant *next = variant->next;
-      unsigned i;
 
-      for (i = 0; i < Elements(variant->function); i++) {
-         if (variant->function[i]) {
-            if (variant->jit_function[i])
-               LLVMFreeMachineCodeForFunction(screen->engine,
-                                              variant->function[i]);
-            LLVMDeleteFunction(variant->function[i]);
-         }
-      }
+   llvmpipe_flush(pipe, 0, &fence);
 
-      FREE(variant);
+   if (fence) {
+      pipe->screen->fence_finish(pipe->screen, fence, 0);
+      pipe->screen->fence_reference(pipe->screen, &fence, NULL);
+   }
 
-      variant = next;
+   li = first_elem(&shader->variants);
+   while(!at_end(&shader->variants, li)) {
+      remove_shader_variant(llvmpipe, li->base);
+      li = next_elem(li);
    }
 
+   assert(shader->variants_cached == 0);
    FREE((void *) shader->base.tokens);
    FREE(shader);
 }
@@ -1215,7 +1237,6 @@ make_variant_key(struct llvmpipe_context *lp,
          lp_sampler_static_state(&key->sampler[i], lp->fragment_sampler_views[i], lp->sampler[i]);
 }
 
-
 /**
  * Update fragment state.  This is called just prior to drawing
  * something when some fragment-related state has changed.
@@ -1225,21 +1246,47 @@ llvmpipe_update_fs(struct llvmpipe_context *lp)
 {
    struct lp_fragment_shader *shader = lp->fs;
    struct lp_fragment_shader_variant_key key;
-   struct lp_fragment_shader_variant *variant;
+   struct lp_fragment_shader_variant *variant = NULL;
+   struct lp_fs_variant_list_item *li;
 
    make_variant_key(lp, shader, &key);
 
-   variant = shader->variants;
-   while(variant) {
-      if(memcmp(&variant->key, &key, sizeof key) == 0)
+   li = first_elem(&shader->variants);
+   while(!at_end(&shader->variants, li)) {
+      if(memcmp(&li->base->key, &key, sizeof key) == 0) {
+         variant = li->base;
          break;
-
-      variant = variant->next;
+      }
+      li = next_elem(li);
    }
 
-   if (!variant) {
+   if (variant) {
+      move_to_head(&lp->fs_variants_list, &variant->list_item_global);
+   }
+   else {
       int64_t t0, t1;
       int64_t dt;
+      unsigned i;
+      if (lp->nr_fs_variants >= LP_MAX_SHADER_VARIANTS) {
+         struct pipe_context *pipe = &lp->pipe;
+         struct pipe_fence_handle *fence = NULL;
+
+         /*
+          * XXX: we need to flush the context until we have some sort of reference
+          * counting in fragment shaders as they may still be binned
+          * Flushing alone might not be sufficient we need to wait on it too.
+          */
+         llvmpipe_flush(pipe, 0, &fence);
+
+         if (fence) {
+            pipe->screen->fence_finish(pipe->screen, fence, 0);
+            pipe->screen->fence_reference(pipe->screen, &fence, NULL);
+         }
+         for (i = 0; i < LP_MAX_SHADER_VARIANTS / 4; i++) {
+            struct lp_fs_variant_list_item *item = last_elem(&lp->fs_variants_list);
+            remove_shader_variant(lp, item->base);
+         }
+      }
       t0 = os_time_get();
 
       variant = generate_variant(lp, shader, &key);
@@ -1248,6 +1295,13 @@ llvmpipe_update_fs(struct llvmpipe_context *lp)
       dt = t1 - t0;
       LP_COUNT_ADD(llvm_compile_time, dt);
       LP_COUNT_ADD(nr_llvm_compiles, 2);  /* emit vs. omit in/out test */
+
+      if (variant) {
+         insert_at_head(&shader->variants, &variant->list_item_local);
+         insert_at_head(&lp->fs_variants_list, &variant->list_item_global);
+         lp->nr_fs_variants++;
+         shader->variants_cached++;
+      }
    }
 
    lp_setup_set_fs_variant(lp->setup, variant);
diff --git a/src/gallium/drivers/llvmpipe/lp_state_fs.h b/src/gallium/drivers/llvmpipe/lp_state_fs.h
index 64ead2a9973..272c9269eae 100644
--- a/src/gallium/drivers/llvmpipe/lp_state_fs.h
+++ b/src/gallium/drivers/llvmpipe/lp_state_fs.h
@@ -64,10 +64,16 @@ struct lp_fragment_shader_variant_key
    struct lp_sampler_static_state sampler[PIPE_MAX_SAMPLERS];
 };
 
+struct lp_fs_variant_list_item
+{
+   struct lp_fragment_shader_variant *base;
+   struct lp_fs_variant_list_item *next, *prev;
+};
 
 struct lp_fragment_shader_variant
 {
    struct lp_fragment_shader_variant_key key;
+   struct lp_fragment_shader *lpfs;
 
    boolean opaque;
 
@@ -75,7 +81,7 @@ struct lp_fragment_shader_variant
 
    lp_jit_frag_func jit_function[2];
 
-   struct lp_fragment_shader_variant *next;
+   struct lp_fs_variant_list_item list_item_global, list_item_local;
 
    /* For debugging/profiling purposes */
    unsigned no;
@@ -89,11 +95,12 @@ struct lp_fragment_shader
 
    struct tgsi_shader_info info;
 
-   struct lp_fragment_shader_variant *variants;
+   struct lp_fs_variant_list_item variants;
 
    /* For debugging/profiling purposes */
    unsigned no;
-   unsigned variant_no;
+   unsigned variants_created;
+   unsigned variants_cached;
 };