Allow reducing the size of the allocated mlock pool via env variable

(BOTAN_MLOCK_POOL_SIZE, specified in decimal KB). Currently we read
this even when setuid as the worst a user could do is disable mlock,
which they can already do via ulimits.

2 files changed, 36 insertions, 9 deletions

diff --git a/doc/relnotes/1_11_14.rst b/doc/relnotes/1_11_14.rst
index 600f6dae1..f3a2527ad 100644
--- a/doc/relnotes/1_11_14.rst
+++ b/doc/relnotes/1_11_14.rst
@@ -7,3 +7,12 @@ Version 1.11.14, Not Yet Released
   finalized is not yet enabled by the default policy, and the
   ciphersuite numbers used are in the experimental range and may
   conflict with other uses.
+
+* The memory allocator available on Unix systems which uses mmap and
+  mlock to lock a pool of memory now checks an environment variable
+  BOTAN_MLOCK_POOL_SIZE. If this is set to a smaller value then the
+  library would originally have allocated the user specified size is
+  used. You can also set it to zero to disable the pool entirely.
+  Previously the allocator would consume all available mlocked memory,
+  this allows botan to coexist with an application which wants to
+  mlock memory of its own.
diff --git a/src/lib/alloc/locking_allocator/locking_allocator.cpp b/src/lib/alloc/locking_allocator/locking_allocator.cpp
index 643868c76..4a3dd3c4c 100644
--- a/src/lib/alloc/locking_allocator/locking_allocator.cpp
+++ b/src/lib/alloc/locking_allocator/locking_allocator.cpp
@@ -23,6 +23,21 @@ namespace {
 */
 const size_t ALIGNMENT_MULTIPLE = 2;
 
+size_t reset_mlock_limit(size_t max_req)
+   {
+   struct rlimit limits;
+   ::getrlimit(RLIMIT_MEMLOCK, &limits);
+
+   if(limits.rlim_cur < limits.rlim_max)
+      {
+      limits.rlim_cur = limits.rlim_max;
+      ::setrlimit(RLIMIT_MEMLOCK, &limits);
+      ::getrlimit(RLIMIT_MEMLOCK, &limits);
+      }
+
+   return std::min<size_t>(limits.rlim_cur, max_req);
+   }
+
 size_t mlock_limit()
    {
    /*
@@ -34,19 +49,22 @@ size_t mlock_limit()
    * programs), but small enough that we should not cause problems
    * even if many processes are mlocking on the same machine.
    */
-   const size_t MLOCK_UPPER_BOUND = 512*1024;
-
-   struct rlimit limits;
-   ::getrlimit(RLIMIT_MEMLOCK, &limits);
+   size_t mlock_requested = 512;
 
-   if(limits.rlim_cur < limits.rlim_max)
+   /*
+   * Allow override via env variable
+   */
+   if(const char* env = ::getenv("BOTAN_MLOCK_POOL_SIZE"))
       {
-      limits.rlim_cur = limits.rlim_max;
-      ::setrlimit(RLIMIT_MEMLOCK, &limits);
-      ::getrlimit(RLIMIT_MEMLOCK, &limits);
+      try
+         {
+         const size_t user_req = std::stoul(env, nullptr);
+         mlock_requested = std::min(user_req, mlock_requested);
+         }
+      catch(std::exception&) { /* ignore it */ }
       }
 
-   return std::min<size_t>(limits.rlim_cur, MLOCK_UPPER_BOUND);
+   return reset_mlock_limit(mlock_requested*1024);
    }
 
 bool ptr_in_pool(const void* pool_ptr, size_t poolsize,