Add another Ed25519 helper to save a few hundred redundant lines

No impact on performance.

3 files changed, 106 insertions, 301 deletions

diff --git a/src/lib/pubkey/ed25519/ed25519_internal.h b/src/lib/pubkey/ed25519/ed25519_internal.h
index 0efeee6d7..cb67a43fd 100644
--- a/src/lib/pubkey/ed25519/ed25519_internal.h
+++ b/src/lib/pubkey/ed25519/ed25519_internal.h
@@ -62,6 +62,23 @@ inline void carry0(int32_t& h0, int32_t& h1)
    h0 -= c * X1;
    }
 
+inline void redc_mul(int64_t& s1,
+                     int64_t& s2,
+                     int64_t& s3,
+                     int64_t& s4,
+                     int64_t& s5,
+                     int64_t& s6,
+                     int64_t& X)
+   {
+   s1 += X * 666643;
+   s2 += X * 470296;
+   s3 += X * 654183;
+   s4 -= X * 997805;
+   s5 += X * 136657;
+   s6 -= X * 683901;
+   X = 0;
+   }
+
 /*
 ge means group element.
 
diff --git a/src/lib/pubkey/ed25519/sc_muladd.cpp b/src/lib/pubkey/ed25519/sc_muladd.cpp
index 948fdcc86..e8d3a0cd5 100644
--- a/src/lib/pubkey/ed25519/sc_muladd.cpp
+++ b/src/lib/pubkey/ed25519/sc_muladd.cpp
@@ -30,91 +30,67 @@ void sc_muladd(uint8_t* s, const uint8_t* a, const uint8_t* b, const uint8_t* c)
 
    const int32_t MASK = 0x1fffff;
 
-   int64_t a0 = MASK & load_3(a);
-   int64_t a1 = MASK & (load_4(a + 2) >> 5);
-   int64_t a2 = MASK & (load_3(a + 5) >> 2);
-   int64_t a3 = MASK & (load_4(a + 7) >> 7);
-   int64_t a4 = MASK & (load_4(a + 10) >> 4);
-   int64_t a5 = MASK & (load_3(a + 13) >> 1);
-   int64_t a6 = MASK & (load_4(a + 15) >> 6);
-   int64_t a7 = MASK & (load_3(a + 18) >> 3);
-   int64_t a8 = MASK & load_3(a + 21);
-   int64_t a9 = MASK & (load_4(a + 23) >> 5);
-   int64_t a10 = MASK & (load_3(a + 26) >> 2);
-   int64_t a11 = (load_4(a + 28) >> 7);
-   int64_t b0 = MASK & load_3(b);
-   int64_t b1 = MASK & (load_4(b + 2) >> 5);
-   int64_t b2 = MASK & (load_3(b + 5) >> 2);
-   int64_t b3 = MASK & (load_4(b + 7) >> 7);
-   int64_t b4 = MASK & (load_4(b + 10) >> 4);
-   int64_t b5 = MASK & (load_3(b + 13) >> 1);
-   int64_t b6 = MASK & (load_4(b + 15) >> 6);
-   int64_t b7 = MASK & (load_3(b + 18) >> 3);
-   int64_t b8 = MASK & load_3(b + 21);
-   int64_t b9 = MASK & (load_4(b + 23) >> 5);
-   int64_t b10 = MASK & (load_3(b + 26) >> 2);
-   int64_t b11 = (load_4(b + 28) >> 7);
-   int64_t c0 = MASK & load_3(c);
-   int64_t c1 = MASK & (load_4(c + 2) >> 5);
-   int64_t c2 = MASK & (load_3(c + 5) >> 2);
-   int64_t c3 = MASK & (load_4(c + 7) >> 7);
-   int64_t c4 = MASK & (load_4(c + 10) >> 4);
-   int64_t c5 = MASK & (load_3(c + 13) >> 1);
-   int64_t c6 = MASK & (load_4(c + 15) >> 6);
-   int64_t c7 = MASK & (load_3(c + 18) >> 3);
-   int64_t c8 = MASK & load_3(c + 21);
-   int64_t c9 = MASK & (load_4(c + 23) >> 5);
-   int64_t c10 = MASK & (load_3(c + 26) >> 2);
-   int64_t c11 = (load_4(c + 28) >> 7);
-   int64_t s0;
-   int64_t s1;
-   int64_t s2;
-   int64_t s3;
-   int64_t s4;
-   int64_t s5;
-   int64_t s6;
-   int64_t s7;
-   int64_t s8;
-   int64_t s9;
-   int64_t s10;
-   int64_t s11;
-   int64_t s12;
-   int64_t s13;
-   int64_t s14;
-   int64_t s15;
-   int64_t s16;
-   int64_t s17;
-   int64_t s18;
-   int64_t s19;
-   int64_t s20;
-   int64_t s21;
-   int64_t s22;
-   int64_t s23;
-
-   s0 = c0 + a0*b0;
-   s1 = c1 + a0*b1 + a1*b0;
-   s2 = c2 + a0*b2 + a1*b1 + a2*b0;
-   s3 = c3 + a0*b3 + a1*b2 + a2*b1 + a3*b0;
-   s4 = c4 + a0*b4 + a1*b3 + a2*b2 + a3*b1 + a4*b0;
-   s5 = c5 + a0*b5 + a1*b4 + a2*b3 + a3*b2 + a4*b1 + a5*b0;
-   s6 = c6 + a0*b6 + a1*b5 + a2*b4 + a3*b3 + a4*b2 + a5*b1 + a6*b0;
-   s7 = c7 + a0*b7 + a1*b6 + a2*b5 + a3*b4 + a4*b3 + a5*b2 + a6*b1 + a7*b0;
-   s8 = c8 + a0*b8 + a1*b7 + a2*b6 + a3*b5 + a4*b4 + a5*b3 + a6*b2 + a7*b1 + a8*b0;
-   s9 = c9 + a0*b9 + a1*b8 + a2*b7 + a3*b6 + a4*b5 + a5*b4 + a6*b3 + a7*b2 + a8*b1 + a9*b0;
-   s10 = c10 + a0*b10 + a1*b9 + a2*b8 + a3*b7 + a4*b6 + a5*b5 + a6*b4 + a7*b3 + a8*b2 + a9*b1 + a10*b0;
-   s11 = c11 + a0*b11 + a1*b10 + a2*b9 + a3*b8 + a4*b7 + a5*b6 + a6*b5 + a7*b4 + a8*b3 + a9*b2 + a10*b1 + a11*b0;
-   s12 = a1*b11 + a2*b10 + a3*b9 + a4*b8 + a5*b7 + a6*b6 + a7*b5 + a8*b4 + a9*b3 + a10*b2 + a11*b1;
-   s13 = a2*b11 + a3*b10 + a4*b9 + a5*b8 + a6*b7 + a7*b6 + a8*b5 + a9*b4 + a10*b3 + a11*b2;
-   s14 = a3*b11 + a4*b10 + a5*b9 + a6*b8 + a7*b7 + a8*b6 + a9*b5 + a10*b4 + a11*b3;
-   s15 = a4*b11 + a5*b10 + a6*b9 + a7*b8 + a8*b7 + a9*b6 + a10*b5 + a11*b4;
-   s16 = a5*b11 + a6*b10 + a7*b9 + a8*b8 + a9*b7 + a10*b6 + a11*b5;
-   s17 = a6*b11 + a7*b10 + a8*b9 + a9*b8 + a10*b7 + a11*b6;
-   s18 = a7*b11 + a8*b10 + a9*b9 + a10*b8 + a11*b7;
-   s19 = a8*b11 + a9*b10 + a10*b9 + a11*b8;
-   s20 = a9*b11 + a10*b10 + a11*b9;
-   s21 = a10*b11 + a11*b10;
-   s22 = a11*b11;
-   s23 = 0;
+   const int64_t a0 = MASK & load_3(a);
+   const int64_t a1 = MASK & (load_4(a + 2) >> 5);
+   const int64_t a2 = MASK & (load_3(a + 5) >> 2);
+   const int64_t a3 = MASK & (load_4(a + 7) >> 7);
+   const int64_t a4 = MASK & (load_4(a + 10) >> 4);
+   const int64_t a5 = MASK & (load_3(a + 13) >> 1);
+   const int64_t a6 = MASK & (load_4(a + 15) >> 6);
+   const int64_t a7 = MASK & (load_3(a + 18) >> 3);
+   const int64_t a8 = MASK & load_3(a + 21);
+   const int64_t a9 = MASK & (load_4(a + 23) >> 5);
+   const int64_t a10 = MASK & (load_3(a + 26) >> 2);
+   const int64_t a11 = (load_4(a + 28) >> 7);
+   const int64_t b0 = MASK & load_3(b);
+   const int64_t b1 = MASK & (load_4(b + 2) >> 5);
+   const int64_t b2 = MASK & (load_3(b + 5) >> 2);
+   const int64_t b3 = MASK & (load_4(b + 7) >> 7);
+   const int64_t b4 = MASK & (load_4(b + 10) >> 4);
+   const int64_t b5 = MASK & (load_3(b + 13) >> 1);
+   const int64_t b6 = MASK & (load_4(b + 15) >> 6);
+   const int64_t b7 = MASK & (load_3(b + 18) >> 3);
+   const int64_t b8 = MASK & load_3(b + 21);
+   const int64_t b9 = MASK & (load_4(b + 23) >> 5);
+   const int64_t b10 = MASK & (load_3(b + 26) >> 2);
+   const int64_t b11 = (load_4(b + 28) >> 7);
+   const int64_t c0 = MASK & load_3(c);
+   const int64_t c1 = MASK & (load_4(c + 2) >> 5);
+   const int64_t c2 = MASK & (load_3(c + 5) >> 2);
+   const int64_t c3 = MASK & (load_4(c + 7) >> 7);
+   const int64_t c4 = MASK & (load_4(c + 10) >> 4);
+   const int64_t c5 = MASK & (load_3(c + 13) >> 1);
+   const int64_t c6 = MASK & (load_4(c + 15) >> 6);
+   const int64_t c7 = MASK & (load_3(c + 18) >> 3);
+   const int64_t c8 = MASK & load_3(c + 21);
+   const int64_t c9 = MASK & (load_4(c + 23) >> 5);
+   const int64_t c10 = MASK & (load_3(c + 26) >> 2);
+   const int64_t c11 = (load_4(c + 28) >> 7);
+
+   int64_t s0 = c0 + a0*b0;
+   int64_t s1 = c1 + a0*b1 + a1*b0;
+   int64_t s2 = c2 + a0*b2 + a1*b1 + a2*b0;
+   int64_t s3 = c3 + a0*b3 + a1*b2 + a2*b1 + a3*b0;
+   int64_t s4 = c4 + a0*b4 + a1*b3 + a2*b2 + a3*b1 + a4*b0;
+   int64_t s5 = c5 + a0*b5 + a1*b4 + a2*b3 + a3*b2 + a4*b1 + a5*b0;
+   int64_t s6 = c6 + a0*b6 + a1*b5 + a2*b4 + a3*b3 + a4*b2 + a5*b1 + a6*b0;
+   int64_t s7 = c7 + a0*b7 + a1*b6 + a2*b5 + a3*b4 + a4*b3 + a5*b2 + a6*b1 + a7*b0;
+   int64_t s8 = c8 + a0*b8 + a1*b7 + a2*b6 + a3*b5 + a4*b4 + a5*b3 + a6*b2 + a7*b1 + a8*b0;
+   int64_t s9 = c9 + a0*b9 + a1*b8 + a2*b7 + a3*b6 + a4*b5 + a5*b4 + a6*b3 + a7*b2 + a8*b1 + a9*b0;
+   int64_t s10 = c10 + a0*b10 + a1*b9 + a2*b8 + a3*b7 + a4*b6 + a5*b5 + a6*b4 + a7*b3 + a8*b2 + a9*b1 + a10*b0;
+   int64_t s11 = c11 + a0*b11 + a1*b10 + a2*b9 + a3*b8 + a4*b7 + a5*b6 + a6*b5 + a7*b4 + a8*b3 + a9*b2 + a10*b1 + a11*b0;
+   int64_t s12 = a1*b11 + a2*b10 + a3*b9 + a4*b8 + a5*b7 + a6*b6 + a7*b5 + a8*b4 + a9*b3 + a10*b2 + a11*b1;
+   int64_t s13 = a2*b11 + a3*b10 + a4*b9 + a5*b8 + a6*b7 + a7*b6 + a8*b5 + a9*b4 + a10*b3 + a11*b2;
+   int64_t s14 = a3*b11 + a4*b10 + a5*b9 + a6*b8 + a7*b7 + a8*b6 + a9*b5 + a10*b4 + a11*b3;
+   int64_t s15 = a4*b11 + a5*b10 + a6*b9 + a7*b8 + a8*b7 + a9*b6 + a10*b5 + a11*b4;
+   int64_t s16 = a5*b11 + a6*b10 + a7*b9 + a8*b8 + a9*b7 + a10*b6 + a11*b5;
+   int64_t s17 = a6*b11 + a7*b10 + a8*b9 + a9*b8 + a10*b7 + a11*b6;
+   int64_t s18 = a7*b11 + a8*b10 + a9*b9 + a10*b8 + a11*b7;
+   int64_t s19 = a8*b11 + a9*b10 + a10*b9 + a11*b8;
+   int64_t s20 = a9*b11 + a10*b10 + a11*b9;
+   int64_t s21 = a10*b11 + a11*b10;
+   int64_t s22 = a11*b11;
+   int64_t s23 = 0;
 
    carry<21>(s0, s1);
    carry<21>(s2, s3);
@@ -141,53 +117,12 @@ void sc_muladd(uint8_t* s, const uint8_t* a, const uint8_t* b, const uint8_t* c)
    carry<21>(s19, s20);
    carry<21>(s21, s22);
 
-   s11 += s23 * 666643;
-   s12 += s23 * 470296;
-   s13 += s23 * 654183;
-   s14 -= s23 * 997805;
-   s15 += s23 * 136657;
-   s16 -= s23 * 683901;
-   s23 = 0;
-
-   s10 += s22 * 666643;
-   s11 += s22 * 470296;
-   s12 += s22 * 654183;
-   s13 -= s22 * 997805;
-   s14 += s22 * 136657;
-   s15 -= s22 * 683901;
-   s22 = 0;
-
-   s9 += s21 * 666643;
-   s10 += s21 * 470296;
-   s11 += s21 * 654183;
-   s12 -= s21 * 997805;
-   s13 += s21 * 136657;
-   s14 -= s21 * 683901;
-   s21 = 0;
-
-   s8 += s20 * 666643;
-   s9 += s20 * 470296;
-   s10 += s20 * 654183;
-   s11 -= s20 * 997805;
-   s12 += s20 * 136657;
-   s13 -= s20 * 683901;
-   s20 = 0;
-
-   s7 += s19 * 666643;
-   s8 += s19 * 470296;
-   s9 += s19 * 654183;
-   s10 -= s19 * 997805;
-   s11 += s19 * 136657;
-   s12 -= s19 * 683901;
-   s19 = 0;
-
-   s6 += s18 * 666643;
-   s7 += s18 * 470296;
-   s8 += s18 * 654183;
-   s9 -= s18 * 997805;
-   s10 += s18 * 136657;
-   s11 -= s18 * 683901;
-   s18 = 0;
+   redc_mul(s11, s12, s13, s14, s15, s16, s23);
+   redc_mul(s10, s11, s12, s13, s14, s15, s22);
+   redc_mul( s9, s10, s11, s12, s13, s14, s21);
+   redc_mul( s8,  s9, s10, s11, s12, s13, s20);
+   redc_mul( s7,  s8,  s9, s10, s11, s12, s19);
+   redc_mul( s6,  s7,  s8,  s9, s10, s11, s18);
 
    carry<21>(s6, s7);
    carry<21>(s8, s9);
@@ -202,53 +137,12 @@ void sc_muladd(uint8_t* s, const uint8_t* a, const uint8_t* b, const uint8_t* c)
    carry<21>(s13, s14);
    carry<21>(s15, s16);
 
-   s5 += s17 * 666643;
-   s6 += s17 * 470296;
-   s7 += s17 * 654183;
-   s8 -= s17 * 997805;
-   s9 += s17 * 136657;
-   s10 -= s17 * 683901;
-   s17 = 0;
-
-   s4 += s16 * 666643;
-   s5 += s16 * 470296;
-   s6 += s16 * 654183;
-   s7 -= s16 * 997805;
-   s8 += s16 * 136657;
-   s9 -= s16 * 683901;
-   s16 = 0;
-
-   s3 += s15 * 666643;
-   s4 += s15 * 470296;
-   s5 += s15 * 654183;
-   s6 -= s15 * 997805;
-   s7 += s15 * 136657;
-   s8 -= s15 * 683901;
-   s15 = 0;
-
-   s2 += s14 * 666643;
-   s3 += s14 * 470296;
-   s4 += s14 * 654183;
-   s5 -= s14 * 997805;
-   s6 += s14 * 136657;
-   s7 -= s14 * 683901;
-   s14 = 0;
-
-   s1 += s13 * 666643;
-   s2 += s13 * 470296;
-   s3 += s13 * 654183;
-   s4 -= s13 * 997805;
-   s5 += s13 * 136657;
-   s6 -= s13 * 683901;
-   s13 = 0;
-
-   s0 += s12 * 666643;
-   s1 += s12 * 470296;
-   s2 += s12 * 654183;
-   s3 -= s12 * 997805;
-   s4 += s12 * 136657;
-   s5 -= s12 * 683901;
-   s12 = 0;
+   redc_mul(s5, s6, s7, s8, s9, s10, s17);
+   redc_mul(s4, s5, s6, s7, s8, s9, s16);
+   redc_mul(s3, s4, s5, s6, s7, s8, s15);
+   redc_mul(s2, s3, s4, s5, s6, s7, s14);
+   redc_mul(s1, s2, s3, s4, s5, s6, s13);
+   redc_mul(s0, s1, s2, s3, s4, s5, s12);
 
    carry<21>(s0, s1);
    carry<21>(s2, s3);
@@ -264,13 +158,7 @@ void sc_muladd(uint8_t* s, const uint8_t* a, const uint8_t* b, const uint8_t* c)
    carry<21>(s9, s10);
    carry<21>(s11, s12);
 
-   s0 += s12 * 666643;
-   s1 += s12 * 470296;
-   s2 += s12 * 654183;
-   s3 -= s12 * 997805;
-   s4 += s12 * 136657;
-   s5 -= s12 * 683901;
-   s12 = 0;
+   redc_mul(s0, s1, s2, s3, s4, s5, s12);
 
    carry<21>(s0, s1);
    carry<21>(s1, s2);
@@ -285,13 +173,7 @@ void sc_muladd(uint8_t* s, const uint8_t* a, const uint8_t* b, const uint8_t* c)
    carry<21>(s10, s11);
    carry0<21>(s11, s12);
 
-   s0 += s12 * 666643;
-   s1 += s12 * 470296;
-   s2 += s12 * 654183;
-   s3 -= s12 * 997805;
-   s4 += s12 * 136657;
-   s5 -= s12 * 683901;
-   s12 = 0;
+   redc_mul(s0, s1, s2, s3, s4, s5, s12);
 
    carry0<21>(s0, s1);
    carry0<21>(s1, s2);
diff --git a/src/lib/pubkey/ed25519/sc_reduce.cpp b/src/lib/pubkey/ed25519/sc_reduce.cpp
index ec8f94955..b9d0f9527 100644
--- a/src/lib/pubkey/ed25519/sc_reduce.cpp
+++ b/src/lib/pubkey/ed25519/sc_reduce.cpp
@@ -51,53 +51,12 @@ void sc_reduce(uint8_t* s)
    int64_t s22 = MASK & (load_4(s + 57) >> 6);
    int64_t s23 = (load_4(s + 60) >> 3);
 
-   s11 += s23 * 666643;
-   s12 += s23 * 470296;
-   s13 += s23 * 654183;
-   s14 -= s23 * 997805;
-   s15 += s23 * 136657;
-   s16 -= s23 * 683901;
-   s23 = 0;
-
-   s10 += s22 * 666643;
-   s11 += s22 * 470296;
-   s12 += s22 * 654183;
-   s13 -= s22 * 997805;
-   s14 += s22 * 136657;
-   s15 -= s22 * 683901;
-   s22 = 0;
-
-   s9 += s21 * 666643;
-   s10 += s21 * 470296;
-   s11 += s21 * 654183;
-   s12 -= s21 * 997805;
-   s13 += s21 * 136657;
-   s14 -= s21 * 683901;
-   s21 = 0;
-
-   s8 += s20 * 666643;
-   s9 += s20 * 470296;
-   s10 += s20 * 654183;
-   s11 -= s20 * 997805;
-   s12 += s20 * 136657;
-   s13 -= s20 * 683901;
-   s20 = 0;
-
-   s7 += s19 * 666643;
-   s8 += s19 * 470296;
-   s9 += s19 * 654183;
-   s10 -= s19 * 997805;
-   s11 += s19 * 136657;
-   s12 -= s19 * 683901;
-   s19 = 0;
-
-   s6 += s18 * 666643;
-   s7 += s18 * 470296;
-   s8 += s18 * 654183;
-   s9 -= s18 * 997805;
-   s10 += s18 * 136657;
-   s11 -= s18 * 683901;
-   s18 = 0;
+   redc_mul(s11, s12, s13, s14, s15, s16, s23);
+   redc_mul(s10, s11, s12, s13, s14, s15, s22);
+   redc_mul( s9, s10, s11, s12, s13, s14, s21);
+   redc_mul( s8,  s9, s10, s11, s12, s13, s20);
+   redc_mul( s7,  s8,  s9, s10, s11, s12, s19);
+   redc_mul( s6,  s7,  s8,  s9, s10, s11, s18);
 
    carry<21>(s6, s7);
    carry<21>(s8, s9);
@@ -112,53 +71,12 @@ void sc_reduce(uint8_t* s)
    carry<21>(s13, s14);
    carry<21>(s15, s16);
 
-   s5 += s17 * 666643;
-   s6 += s17 * 470296;
-   s7 += s17 * 654183;
-   s8 -= s17 * 997805;
-   s9 += s17 * 136657;
-   s10 -= s17 * 683901;
-   s17 = 0;
-
-   s4 += s16 * 666643;
-   s5 += s16 * 470296;
-   s6 += s16 * 654183;
-   s7 -= s16 * 997805;
-   s8 += s16 * 136657;
-   s9 -= s16 * 683901;
-   s16 = 0;
-
-   s3 += s15 * 666643;
-   s4 += s15 * 470296;
-   s5 += s15 * 654183;
-   s6 -= s15 * 997805;
-   s7 += s15 * 136657;
-   s8 -= s15 * 683901;
-   s15 = 0;
-
-   s2 += s14 * 666643;
-   s3 += s14 * 470296;
-   s4 += s14 * 654183;
-   s5 -= s14 * 997805;
-   s6 += s14 * 136657;
-   s7 -= s14 * 683901;
-   s14 = 0;
-
-   s1 += s13 * 666643;
-   s2 += s13 * 470296;
-   s3 += s13 * 654183;
-   s4 -= s13 * 997805;
-   s5 += s13 * 136657;
-   s6 -= s13 * 683901;
-   s13 = 0;
-
-   s0 += s12 * 666643;
-   s1 += s12 * 470296;
-   s2 += s12 * 654183;
-   s3 -= s12 * 997805;
-   s4 += s12 * 136657;
-   s5 -= s12 * 683901;
-   s12 = 0;
+   redc_mul(s5, s6, s7, s8, s9, s10, s17);
+   redc_mul(s4, s5, s6, s7, s8, s9, s16);
+   redc_mul(s3, s4, s5, s6, s7, s8, s15);
+   redc_mul(s2, s3, s4, s5, s6, s7, s14);
+   redc_mul(s1, s2, s3, s4, s5, s6, s13);
+   redc_mul(s0, s1, s2, s3, s4, s5, s12);
 
    carry<21>(s0, s1);
    carry<21>(s2, s3);
@@ -174,13 +92,7 @@ void sc_reduce(uint8_t* s)
    carry<21>(s9, s10);
    carry<21>(s11, s12);
 
-   s0 += s12 * 666643;
-   s1 += s12 * 470296;
-   s2 += s12 * 654183;
-   s3 -= s12 * 997805;
-   s4 += s12 * 136657;
-   s5 -= s12 * 683901;
-   s12 = 0;
+   redc_mul(s0, s1, s2, s3, s4, s5, s12);
 
    carry<21>(s0, s1);
    carry<21>(s1, s2);
@@ -195,13 +107,7 @@ void sc_reduce(uint8_t* s)
    carry<21>(s10, s11);
    carry0<21>(s11, s12);
 
-   s0 += s12 * 666643;
-   s1 += s12 * 470296;
-   s2 += s12 * 654183;
-   s3 -= s12 * 997805;
-   s4 += s12 * 136657;
-   s5 -= s12 * 683901;
-   s12 = 0;
+   redc_mul(s0, s1, s2, s3, s4, s5, s12);
 
    carry0<21>(s0, s1);
    carry0<21>(s1, s2);