util: Use ZSTD for shader cache if possible

This allows ZSTD instead of ZLIB to be used for compressing the shader
cache.

On a 72 core system emulating skl with a full shader-db (with i965):
ZSTD:
    1915.10s user 229.27s system 5150% cpu 41.632 total (cold cache)
    225.40s user 10.87s system 3810% cpu 6.201 total (warm cache)
    154M (235M on disk)
ZLIB:
    2231.33s user 194.24s system 1899% cpu 2:07.72 total (cold cache)
    229.15s user 10.63s system 3906% cpu 6.139 total (warm cache)
    163M (244M on disk)

Tim Arceri sees (8 core ryzen and a full shader-db):
ZSTD:
    2505.22 user 40.50 system 3:18.73 elapsed 1280% CPU (cold cache)
    418.71 user 14.93 system 0:46.53 elapsed 931% CPU (warm cache)
    454.3 MB (681.7 MB on disk)
ZLIB:
    3069.83 user 40.02 system 4:20.13 elapsed 1195% CPU (cold cache)
    425.50 user 15.17 system 0:46.80 elapsed 941% CPU (warm cache)
    470.3 MB (701.4 MB on disk)

Reviewed-by: Eric Engestrom <[email protected]> (v1)
Reviewed-by: Eric Anholt <[email protected]>

2 files changed, 31 insertions, 0 deletions

diff --git a/src/util/disk_cache.c b/src/util/disk_cache.c
index 0cd92ca2d5b..e5b21b9a79f 100644
--- a/src/util/disk_cache.c
+++ b/src/util/disk_cache.c
@@ -40,6 +40,10 @@
 #include <inttypes.h>
 #include "zlib.h"
 
+#ifdef HAVE_ZSTD
+#include "zstd.h"
+#endif
+
 #include "util/crc32.h"
 #include "util/debug.h"
 #include "util/rand_xor.h"
@@ -75,6 +79,9 @@
  */
 #define CACHE_VERSION 1
 
+/* 3 is the recomended level, with 22 as the absolute maximum */
+#define ZSTD_COMPRESSION_LEVEL 3
+
 struct disk_cache {
    /* The path to the cache directory. */
    char *path;
@@ -738,6 +745,23 @@ static size_t
 deflate_and_write_to_disk(const void *in_data, size_t in_data_size, int dest,
                           const char *filename)
 {
+#ifdef HAVE_ZSTD
+   /* from the zstd docs (https://facebook.github.io/zstd/zstd_manual.html):
+    * compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
+    */
+   size_t out_size = ZSTD_compressBound(in_data_size);
+   void * out = malloc(out_size);
+
+   size_t ret = ZSTD_compress(out, out_size, in_data, in_data_size,
+                              ZSTD_COMPRESSION_LEVEL);
+   if (ZSTD_isError(ret)) {
+      free(out);
+      return 0;
+   }
+   write_all(dest, out, ret);
+   free(out);
+   return ret;
+#else
    unsigned char *out;
 
    /* allocate deflate state */
@@ -798,6 +822,7 @@ deflate_and_write_to_disk(const void *in_data, size_t in_data_size, int dest,
    (void)deflateEnd(&strm);
    free(out);
    return compressed_size;
+# endif
 }
 
 static struct disk_cache_put_job *
@@ -1059,6 +1084,10 @@ static bool
 inflate_cache_data(uint8_t *in_data, size_t in_data_size,
                    uint8_t *out_data, size_t out_data_size)
 {
+#ifdef HAVE_ZSTD
+   size_t ret = ZSTD_decompress(out_data, out_data_size, in_data, in_data_size);
+   return !ZSTD_isError(ret);
+#else
    z_stream strm;
 
    /* allocate inflate state */
@@ -1089,6 +1118,7 @@ inflate_cache_data(uint8_t *in_data, size_t in_data_size,
    /* clean up and return */
    (void)inflateEnd(&strm);
    return true;
+#endif
 }
 
 void *
diff --git a/src/util/meson.build b/src/util/meson.build
index 972d757195a..a7990ad52cc 100644
--- a/src/util/meson.build
+++ b/src/util/meson.build
@@ -149,6 +149,7 @@ deps_for_libmesa_util = [
   dep_atomic,
   dep_m,
   dep_valgrind,
+  dep_zstd,
 ]
 
 if with_platform_android