Pass workspace size to various bigint_ functions

These functions made assumptions about the workspace size available,
which if incorrect would cause memory corruption. Since the length is
always available at the caller, just provide it and avoid problems.

3 files changed, 32 insertions, 23 deletions

diff --git a/src/lib/math/mp/mp_core.h b/src/lib/math/mp/mp_core.h
index 06f87015f..9efcec952 100644
--- a/src/lib/math/mp/mp_core.h
+++ b/src/lib/math/mp/mp_core.h
@@ -132,21 +132,22 @@ void bigint_linmul3(word z[], const word x[], size_t x_size, word y);
 void bigint_monty_redc(word z[],
                        const word p[], size_t p_size,
                        word p_dash,
-                       word workspace[]);
+                       word workspace[],
+                       size_t ws_size);
 
 /*
 * Montgomery Multiplication
 */
 void bigint_monty_mul(BigInt& z, const BigInt& x, const BigInt& y,
                       const word p[], size_t p_size, word p_dash,
-                      word workspace[]);
+                      word workspace[], size_t ws_size);
 
 /*
 * Montgomery Squaring
 */
 void bigint_monty_sqr(BigInt& z, const BigInt& x,
                       const word p[], size_t p_size, word p_dash,
-                      word workspace[]);
+                      word workspace[], size_t ws_size);
 
 /**
 * Compare x and y
@@ -182,15 +183,17 @@ void bigint_comba_sqr16(word out[32], const word in[16]);
 /*
 * High Level Multiplication/Squaring Interfaces
 */
-void bigint_mul(BigInt& z, const BigInt& x, const BigInt& y, word workspace[]);
+void bigint_mul(BigInt& z, const BigInt& x, const BigInt& y,
+                word workspace[], size_t ws_size);
 
 void bigint_mul(word z[], size_t z_size,
                 const word x[], size_t x_size, size_t x_sw,
                 const word y[], size_t y_size, size_t y_sw,
-                word workspace[]);
+                word workspace[], size_t ws_size);
 
-void bigint_sqr(word z[], size_t z_size, word workspace[],
-                const word x[], size_t x_size, size_t x_sw);
+void bigint_sqr(word z[], size_t z_size,
+                const word x[], size_t x_size, size_t x_sw,
+                word workspace[], size_t ws_size);
 
 }
 
diff --git a/src/lib/math/mp/mp_karat.cpp b/src/lib/math/mp/mp_karat.cpp
index 8348025d9..fb8b16161 100644
--- a/src/lib/math/mp/mp_karat.cpp
+++ b/src/lib/math/mp/mp_karat.cpp
@@ -250,18 +250,19 @@ size_t karatsuba_size(size_t z_size, size_t x_size, size_t x_sw)
 /*
 * Multiplication Algorithm Dispatcher
 */
-void bigint_mul(BigInt& z, const BigInt& x, const BigInt& y, word workspace[])
+void bigint_mul(BigInt& z, const BigInt& x, const BigInt& y,
+                word workspace[], size_t ws_size)
    {
    return bigint_mul(z.mutable_data(), z.size(),
                      x.data(), x.size(), x.sig_words(),
                      y.data(), y.size(), y.sig_words(),
-                     workspace);
+                     workspace, ws_size);
    }
 
 void bigint_mul(word z[], size_t z_size,
                 const word x[], size_t x_size, size_t x_sw,
                 const word y[], size_t y_size, size_t y_sw,
-                word workspace[])
+                word workspace[], size_t ws_size)
    {
    clear_mem(z, z_size);
 
@@ -308,7 +309,7 @@ void bigint_mul(word z[], size_t z_size,
       {
       const size_t N = karatsuba_size(z_size, x_size, x_sw, y_size, y_sw);
 
-      if(N)
+      if(N && ws_size >= 2*N)
          karatsuba_mul(z, x, y, N, workspace);
       else
          basecase_mul(z, x, x_sw, y, y_sw);
@@ -318,8 +319,9 @@ void bigint_mul(word z[], size_t z_size,
 /*
 * Squaring Algorithm Dispatcher
 */
-void bigint_sqr(word z[], size_t z_size, word workspace[],
-                const word x[], size_t x_size, size_t x_sw)
+void bigint_sqr(word z[], size_t z_size,
+                const word x[], size_t x_size, size_t x_sw,
+                word workspace[], size_t ws_size)
    {
    BOTAN_ASSERT(z_size/2 >= x_sw, "Output size is sufficient");
 
@@ -355,7 +357,7 @@ void bigint_sqr(word z[], size_t z_size, word workspace[],
       {
       const size_t N = karatsuba_size(z_size, x_size, x_sw);
 
-      if(N)
+      if(N && ws_size >= 2*N)
          karatsuba_sqr(z, x, N, workspace);
       else
          basecase_mul(z, x, x_sw, x, x_sw);
diff --git a/src/lib/math/mp/mp_monty.cpp b/src/lib/math/mp/mp_monty.cpp
index 2599266b0..cc6388f4d 100644
--- a/src/lib/math/mp/mp_monty.cpp
+++ b/src/lib/math/mp/mp_monty.cpp
@@ -20,11 +20,14 @@ namespace Botan {
 * Montgomery Reduction Algorithm
 */
 void bigint_monty_redc(word z[],
-                       const word p[], size_t p_size,
-                       word p_dash, word ws[])
+                       const word p[], size_t p_size, word p_dash,
+                       word ws[], size_t ws_size)
    {
    const size_t z_size = 2*(p_size+1);
 
+   if(ws_size < z_size)
+      throw Invalid_Argument("bigint_monty_redc workspace too small");
+
    CT::poison(z, z_size);
    CT::poison(p, p_size);
    CT::poison(ws, 2*(p_size+1));
@@ -96,24 +99,25 @@ void bigint_monty_redc(word z[],
 
 void bigint_monty_mul(BigInt& z, const BigInt& x, const BigInt& y,
                       const word p[], size_t p_size, word p_dash,
-                      word ws[])
+                      word ws[], size_t ws_size)
    {
-   bigint_mul(z, x, y, &ws[0]);
+   bigint_mul(z, x, y, ws, ws_size);
 
    bigint_monty_redc(z.mutable_data(),
                      p, p_size, p_dash,
-                     ws);
+                     ws, ws_size);
    }
 
 void bigint_monty_sqr(BigInt& z, const BigInt& x, const word p[],
-                      size_t p_size, word p_dash, word ws[])
+                      size_t p_size, word p_dash, word ws[], size_t ws_size)
    {
-   bigint_sqr(z.mutable_data(), z.size(), &ws[0],
-              x.data(), x.size(), x.sig_words());
+   bigint_sqr(z.mutable_data(), z.size(),
+              x.data(), x.size(), x.sig_words(),
+              ws, ws_size);
 
    bigint_monty_redc(z.mutable_data(),
                      p, p_size, p_dash,
-                     ws);
+                     ws, ws_size);
    }
 
 }