Fix a bug in Barrett reduction

-x*n % n would reduce to n instead of zero.

Also some small optimizations and cleanups.

1 files changed, 8 insertions, 4 deletions

diff --git a/src/fuzzer/barrett.cpp b/src/fuzzer/barrett.cpp
index 1c5d88f87..09aed517e 100644
--- a/src/fuzzer/barrett.cpp
+++ b/src/fuzzer/barrett.cpp
@@ -12,20 +12,24 @@ void fuzz(const uint8_t in[], size_t len)
    {
    static const size_t max_bits = 2048;
 
-   if(len % 2 != 0)
+   if(len <= 1 || len % 3 != 1)
       return;
 
-   const size_t part_size = len / 2;
+   const size_t part_size = len / 3;
 
    if(part_size * 8 > max_bits)
       return;
 
-   const Botan::BigInt x = Botan::BigInt::decode(in, part_size);
-   const Botan::BigInt p = Botan::BigInt::decode(in + part_size, part_size);
+   uint8_t flags = in[0];
+   Botan::BigInt x = Botan::BigInt::decode(in + 1, part_size * 2);
+   const Botan::BigInt p = Botan::BigInt::decode(in + 1 + part_size * 2, part_size);
 
    if(p.is_zero())
       return;
 
+   if(flags & 1)
+      x.flip_sign();
+
    const Botan::BigInt ref = x % p;
 
    const Botan::Modular_Reducer mod_p(p);