Pass workspace size to various bigint_ functions

These functions made assumptions about the workspace size available,
which if incorrect would cause memory corruption. Since the length is
always available at the caller, just provide it and avoid problems.

8 files changed, 52 insertions, 41 deletions

diff --git a/src/lib/math/bigint/big_ops2.cpp b/src/lib/math/bigint/big_ops2.cpp
index 639d87ceb..2f81989c3 100644
--- a/src/lib/math/bigint/big_ops2.cpp
+++ b/src/lib/math/bigint/big_ops2.cpp
@@ -119,7 +119,7 @@ BigInt& BigInt::operator*=(const BigInt& y)
       {
       grow_to(size() + y.size());
       secure_vector<word> workspace(size());
-      bigint_mul(*this, BigInt(*this), y, workspace.data());
+      bigint_mul(*this, BigInt(*this), y, workspace.data(), workspace.size());
       }
 
    return (*this);
diff --git a/src/lib/math/bigint/big_ops3.cpp b/src/lib/math/bigint/big_ops3.cpp
index 680eca635..8bff790a6 100644
--- a/src/lib/math/bigint/big_ops3.cpp
+++ b/src/lib/math/bigint/big_ops3.cpp
@@ -95,7 +95,7 @@ BigInt operator*(const BigInt& x, const BigInt& y)
    else if(x_sw && y_sw)
       {
       secure_vector<word> workspace(z.size());
-      bigint_mul(z, x, y, workspace.data());
+      bigint_mul(z, x, y, workspace.data(), workspace.size());
       }
 
    if(x_sw && y_sw && x.sign() != y.sign())
diff --git a/src/lib/math/ec_gfp/curve_gfp.cpp b/src/lib/math/ec_gfp/curve_gfp.cpp
index efeb5210f..83d1c841c 100644
--- a/src/lib/math/ec_gfp/curve_gfp.cpp
+++ b/src/lib/math/ec_gfp/curve_gfp.cpp
@@ -88,8 +88,9 @@ void CurveGFp_Montgomery::curve_mul(BigInt& z, const BigInt& x, const BigInt& y,
    z.grow_to(output_size);
    z.clear();
 
-   bigint_monty_mul(z, x, y, m_p.data(), m_p_words, m_p_dash, ws.data());
-
+   bigint_monty_mul(z, x, y,
+                    m_p.data(), m_p_words, m_p_dash,
+                    ws.data(), ws.size());
    }
 
 void CurveGFp_Montgomery::curve_sqr(BigInt& z, const BigInt& x,
@@ -112,7 +113,7 @@ void CurveGFp_Montgomery::curve_sqr(BigInt& z, const BigInt& x,
    z.clear();
 
    bigint_monty_sqr(z, x, m_p.data(), m_p_words, m_p_dash,
-                    ws.data());
+                    ws.data(), ws.size());
    }
 
 class CurveGFp_NIST : public CurveGFp_Repr
@@ -168,7 +169,7 @@ void CurveGFp_NIST::curve_mul(BigInt& z, const BigInt& x, const BigInt& y,
    z.grow_to(output_size);
    z.clear();
 
-   bigint_mul(z, x, y, ws.data());
+   bigint_mul(z, x, y, ws.data(), ws.size());
 
    this->redc(z, ws);
    }
@@ -190,8 +191,9 @@ void CurveGFp_NIST::curve_sqr(BigInt& z, const BigInt& x,
    z.grow_to(output_size);
    z.clear();
 
-   bigint_sqr(z.mutable_data(), output_size, ws.data(),
-              x.data(), x.size(), x.sig_words());
+   bigint_sqr(z.mutable_data(), output_size,
+              x.data(), x.size(), x.sig_words(),
+              ws.data(), ws.size());
 
    this->redc(z, ws);
    }
diff --git a/src/lib/math/mp/mp_core.h b/src/lib/math/mp/mp_core.h
index 06f87015f..9efcec952 100644
--- a/src/lib/math/mp/mp_core.h
+++ b/src/lib/math/mp/mp_core.h
@@ -132,21 +132,22 @@ void bigint_linmul3(word z[], const word x[], size_t x_size, word y);
 void bigint_monty_redc(word z[],
                        const word p[], size_t p_size,
                        word p_dash,
-                       word workspace[]);
+                       word workspace[],
+                       size_t ws_size);
 
 /*
 * Montgomery Multiplication
 */
 void bigint_monty_mul(BigInt& z, const BigInt& x, const BigInt& y,
                       const word p[], size_t p_size, word p_dash,
-                      word workspace[]);
+                      word workspace[], size_t ws_size);
 
 /*
 * Montgomery Squaring
 */
 void bigint_monty_sqr(BigInt& z, const BigInt& x,
                       const word p[], size_t p_size, word p_dash,
-                      word workspace[]);
+                      word workspace[], size_t ws_size);
 
 /**
 * Compare x and y
@@ -182,15 +183,17 @@ void bigint_comba_sqr16(word out[32], const word in[16]);
 /*
 * High Level Multiplication/Squaring Interfaces
 */
-void bigint_mul(BigInt& z, const BigInt& x, const BigInt& y, word workspace[]);
+void bigint_mul(BigInt& z, const BigInt& x, const BigInt& y,
+                word workspace[], size_t ws_size);
 
 void bigint_mul(word z[], size_t z_size,
                 const word x[], size_t x_size, size_t x_sw,
                 const word y[], size_t y_size, size_t y_sw,
-                word workspace[]);
+                word workspace[], size_t ws_size);
 
-void bigint_sqr(word z[], size_t z_size, word workspace[],
-                const word x[], size_t x_size, size_t x_sw);
+void bigint_sqr(word z[], size_t z_size,
+                const word x[], size_t x_size, size_t x_sw,
+                word workspace[], size_t ws_size);
 
 }
 
diff --git a/src/lib/math/mp/mp_karat.cpp b/src/lib/math/mp/mp_karat.cpp
index 8348025d9..fb8b16161 100644
--- a/src/lib/math/mp/mp_karat.cpp
+++ b/src/lib/math/mp/mp_karat.cpp
@@ -250,18 +250,19 @@ size_t karatsuba_size(size_t z_size, size_t x_size, size_t x_sw)
 /*
 * Multiplication Algorithm Dispatcher
 */
-void bigint_mul(BigInt& z, const BigInt& x, const BigInt& y, word workspace[])
+void bigint_mul(BigInt& z, const BigInt& x, const BigInt& y,
+                word workspace[], size_t ws_size)
    {
    return bigint_mul(z.mutable_data(), z.size(),
                      x.data(), x.size(), x.sig_words(),
                      y.data(), y.size(), y.sig_words(),
-                     workspace);
+                     workspace, ws_size);
    }
 
 void bigint_mul(word z[], size_t z_size,
                 const word x[], size_t x_size, size_t x_sw,
                 const word y[], size_t y_size, size_t y_sw,
-                word workspace[])
+                word workspace[], size_t ws_size)
    {
    clear_mem(z, z_size);
 
@@ -308,7 +309,7 @@ void bigint_mul(word z[], size_t z_size,
       {
       const size_t N = karatsuba_size(z_size, x_size, x_sw, y_size, y_sw);
 
-      if(N)
+      if(N && ws_size >= 2*N)
          karatsuba_mul(z, x, y, N, workspace);
       else
          basecase_mul(z, x, x_sw, y, y_sw);
@@ -318,8 +319,9 @@ void bigint_mul(word z[], size_t z_size,
 /*
 * Squaring Algorithm Dispatcher
 */
-void bigint_sqr(word z[], size_t z_size, word workspace[],
-                const word x[], size_t x_size, size_t x_sw)
+void bigint_sqr(word z[], size_t z_size,
+                const word x[], size_t x_size, size_t x_sw,
+                word workspace[], size_t ws_size)
    {
    BOTAN_ASSERT(z_size/2 >= x_sw, "Output size is sufficient");
 
@@ -355,7 +357,7 @@ void bigint_sqr(word z[], size_t z_size, word workspace[],
       {
       const size_t N = karatsuba_size(z_size, x_size, x_sw);
 
-      if(N)
+      if(N && ws_size >= 2*N)
          karatsuba_sqr(z, x, N, workspace);
       else
          basecase_mul(z, x, x_sw, x, x_sw);
diff --git a/src/lib/math/mp/mp_monty.cpp b/src/lib/math/mp/mp_monty.cpp
index 2599266b0..cc6388f4d 100644
--- a/src/lib/math/mp/mp_monty.cpp
+++ b/src/lib/math/mp/mp_monty.cpp
@@ -20,11 +20,14 @@ namespace Botan {
 * Montgomery Reduction Algorithm
 */
 void bigint_monty_redc(word z[],
-                       const word p[], size_t p_size,
-                       word p_dash, word ws[])
+                       const word p[], size_t p_size, word p_dash,
+                       word ws[], size_t ws_size)
    {
    const size_t z_size = 2*(p_size+1);
 
+   if(ws_size < z_size)
+      throw Invalid_Argument("bigint_monty_redc workspace too small");
+
    CT::poison(z, z_size);
    CT::poison(p, p_size);
    CT::poison(ws, 2*(p_size+1));
@@ -96,24 +99,25 @@ void bigint_monty_redc(word z[],
 
 void bigint_monty_mul(BigInt& z, const BigInt& x, const BigInt& y,
                       const word p[], size_t p_size, word p_dash,
-                      word ws[])
+                      word ws[], size_t ws_size)
    {
-   bigint_mul(z, x, y, &ws[0]);
+   bigint_mul(z, x, y, ws, ws_size);
 
    bigint_monty_redc(z.mutable_data(),
                      p, p_size, p_dash,
-                     ws);
+                     ws, ws_size);
    }
 
 void bigint_monty_sqr(BigInt& z, const BigInt& x, const word p[],
-                      size_t p_size, word p_dash, word ws[])
+                      size_t p_size, word p_dash, word ws[], size_t ws_size)
    {
-   bigint_sqr(z.mutable_data(), z.size(), &ws[0],
-              x.data(), x.size(), x.sig_words());
+   bigint_sqr(z.mutable_data(), z.size(),
+              x.data(), x.size(), x.sig_words(),
+              ws, ws_size);
 
    bigint_monty_redc(z.mutable_data(),
                      p, p_size, p_dash,
-                     ws);
+                     ws, ws_size);
    }
 
 }
diff --git a/src/lib/math/numbertheory/monty_exp.cpp b/src/lib/math/numbertheory/monty_exp.cpp
index 35d4cd668..bfb17a87c 100644
--- a/src/lib/math/numbertheory/monty_exp.cpp
+++ b/src/lib/math/numbertheory/monty_exp.cpp
@@ -61,14 +61,14 @@ Montgomery_Exponentation_State::Montgomery_Exponentation_State(const BigInt& g,
 
    bigint_monty_mul(z, m_g[0], m_R2_mod,
                     m_p.data(), m_p_words, m_mod_prime,
-                    workspace.data());
+                    workspace.data(), workspace.size());
    m_g[0] = z;
 
    m_g[1] = mod_p.reduce(g);
 
    bigint_monty_mul(z, m_g[1], m_R2_mod,
                     m_p.data(), m_p_words, m_mod_prime,
-                    workspace.data());
+                    workspace.data(), workspace.size());
 
    m_g[1] = z;
 
@@ -79,7 +79,7 @@ Montgomery_Exponentation_State::Montgomery_Exponentation_State(const BigInt& g,
       const BigInt& y = m_g[i-1];
 
       bigint_monty_mul(z, x, y, m_p.data(), m_p_words, m_mod_prime,
-                       workspace.data());
+                       workspace.data(), workspace.size());
 
       m_g[i] = z;
       m_g[i].shrink_to_fit();
@@ -104,7 +104,7 @@ BigInt Montgomery_Exponentation_State::exponentiation(const BigInt& k) const
       for(size_t j = 0; j != m_window_bits; ++j)
          {
          bigint_monty_sqr(z, x, m_p.data(), m_p_words, m_mod_prime,
-                          workspace.data());
+                          workspace.data(), workspace.size());
 
          x = z;
          }
@@ -116,11 +116,11 @@ BigInt Montgomery_Exponentation_State::exponentiation(const BigInt& k) const
       bigint_mul(z.mutable_data(), z.size(),
                  x.data(), x.size(), x.sig_words(),
                  e.data(), m_p_words, m_p_words,
-                 workspace.data());
+                 workspace.data(), workspace.size());
 
       bigint_monty_redc(z.mutable_data(),
                         m_p.data(), m_p_words, m_mod_prime,
-                        workspace.data());
+                        workspace.data(), workspace.size());
 
       x = z;
       }
@@ -129,7 +129,7 @@ BigInt Montgomery_Exponentation_State::exponentiation(const BigInt& k) const
 
    bigint_monty_redc(x.mutable_data(),
                      m_p.data(), m_p_words, m_mod_prime,
-                     workspace.data());
+                     workspace.data(), workspace.size());
 
    return x;
    }
diff --git a/src/lib/math/numbertheory/mp_numth.cpp b/src/lib/math/numbertheory/mp_numth.cpp
index d6eb39e4a..c39c40520 100644
--- a/src/lib/math/numbertheory/mp_numth.cpp
+++ b/src/lib/math/numbertheory/mp_numth.cpp
@@ -24,8 +24,8 @@ BigInt square(const BigInt& x)
    secure_vector<word> workspace(z.size());
 
    bigint_sqr(z.mutable_data(), z.size(),
-              workspace.data(),
-              x.data(), x.size(), x_sw);
+              x.data(), x.size(), x_sw,
+              workspace.data(), workspace.size());
    return z;
    }
 
@@ -44,7 +44,7 @@ BigInt mul_add(const BigInt& a, const BigInt& b, const BigInt& c)
    BigInt r(sign, std::max(a.size() + b.size(), c.sig_words()) + 1);
    secure_vector<word> workspace(r.size());
 
-   bigint_mul(r, a, b, workspace.data());
+   bigint_mul(r, a, b, workspace.data(), workspace.size());
 
    const size_t r_size = std::max(r.sig_words(), c.sig_words());
    bigint_add2(r.mutable_data(), r_size, c.data(), c.sig_words());