Bug#13802242 MY_DECIMAL::SANITY_CHECK ASSERTION: ASSERTION FAILED: FOO2 == TEST_VALUE

Tor Didriksen · Tor Didriksen · commit 819980fcbc74 · 2012-10-19T16:38:24.000+02:00
Yet another buffer overrun the decimal library.
diff --git a/strings/decimal.c b/strings/decimal.c
@@ -1717,10 +1717,18 @@ decimal_round(const decimal_t *from, decimal_t *to, int scale,
       }
       for (buf1=to->buf + intg0 + MY_MAX(frac0, 0); buf1 > to->buf; buf1--)
       {
-        buf1[0]=buf1[-1];
+        /* Avoid out-of-bounds write. */
+        if (buf1 < to->buf + len)
+          buf1[0]=buf1[-1];
+        else
+          error= E_DEC_OVERFLOW;
       }
       *buf1=1;
-      to->intg++;
+      /* We cannot have more than 9 * 9 = 81 digits. */
+      if (to->intg < len * DIG_PER_DEC1)
+        to->intg++;
+      else
+        error= E_DEC_OVERFLOW;
     }
   }
   else
@@ -1752,6 +1760,7 @@ decimal_round(const decimal_t *from, decimal_t *to, int scale,
     scale=0;
 
 done:
+  DBUG_ASSERT(to->intg <= (len * DIG_PER_DEC1));
   to->frac=scale;
   return error;
 }

Original file line number	Diff line number	Diff line change
`@@ -1717,10 +1717,18 @@ decimal_round(const decimal_t from, decimal_t to, int scale,`
`1717`	`1717`	`}`
`1718`	`1718`	`for (buf1=to->buf + intg0 + MY_MAX(frac0, 0); buf1 > to->buf; buf1--)`
`1719`	`1719`	`{`
`1720`		`- buf1[0]=buf1[-1];`
	`1720`	`+ /* Avoid out-of-bounds write. */`
	`1721`	`+ if (buf1 < to->buf + len)`
	`1722`	`+ buf1[0]=buf1[-1];`
	`1723`	`+ else`
	`1724`	`+ error= E_DEC_OVERFLOW;`
`1721`	`1725`	`}`
`1722`	`1726`	`*buf1=1;`
`1723`		`- to->intg++;`
	`1727`	`+ /* We cannot have more than 9 * 9 = 81 digits. */`
	`1728`	`+ if (to->intg < len * DIG_PER_DEC1)`
	`1729`	`+ to->intg++;`
	`1730`	`+ else`
	`1731`	`+ error= E_DEC_OVERFLOW;`
`1724`	`1732`	`}`
`1725`	`1733`	`}`
`1726`	`1734`	`else`
`@@ -1752,6 +1760,7 @@ decimal_round(const decimal_t from, decimal_t to, int scale,`
`1752`	`1760`	`scale=0;`
`1753`	`1761`
`1754`	`1762`	`done:`
	`1763`	`+ DBUG_ASSERT(to->intg <= (len * DIG_PER_DEC1));`
`1755`	`1764`	`to->frac=scale;`
`1756`	`1765`	`return error;`
`1757`	`1766`	`}`