MDEV-33534 UBSAN: Negation of -X cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_double_round from sql/item_func.cc|

The negation in this line: ulonglong abs_dec= dec_negative ? -dec : dec; did not take into account that 'dec' can be the smallest possible signed negative value -9223372036854775808. Its negation is an operation with an undefined behavior. Fixing the code to use Longlong_hybrid, which implements a safe method to get an absolute value.

Author: abarkov

mysql-test/main/func_math.result

@@ -3617,5 +3617,20 @@ SELECT CRC32(ExtractValue('<a><b/></a>', '/a/b')) AS f;
 f
 0
 #
+# MDEV-33534 UBSAN: Negation of -X cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_double_round from sql/item_func.cc|
+#
+SELECT TRUNCATE(EXP(-1.e-2),-1.e+30) AS c1;
+c1
+0
+SELECT (TRUNCATE(EXP(-1.e-2),-1.e+30) % RADIANS(-1)) AS c1;
+c1
+0
+SELECT (TRUNCATE(EXP(-1.e-2),-1.e+30) % RADIANS(-1)) * (LAST_DAY('1-03-30 1:29:12') MOD 1 + COS(-1)) AS c1;
+c1
+0
+SELECT(ASIN(-1)+ LN(-1)) % (ATAN(-1) MOD FLOOR(1)) * (TRUNCATE(EXP(-1.e-2),-1.e+30) % RADIANS(-1)) * (LAST_DAY('1-03-30 1:29:12') MOD 1 + COS(-1)) AS c1;
+c1
+NULL
+#
 # End of 10.5 tests
 #

mysql-test/main/func_math.test

@@ -1929,6 +1929,16 @@ DROP TABLE t2, t1;
 SELECT CRC32(ExtractValue('<a><b/></a>', '/a/b')) AS f;
 
 
+--echo #
+--echo # MDEV-33534 UBSAN: Negation of -X cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_double_round from sql/item_func.cc|
+--echo #
+
+SELECT TRUNCATE(EXP(-1.e-2),-1.e+30) AS c1;
+SELECT (TRUNCATE(EXP(-1.e-2),-1.e+30) % RADIANS(-1)) AS c1;
+SELECT (TRUNCATE(EXP(-1.e-2),-1.e+30) % RADIANS(-1)) * (LAST_DAY('1-03-30 1:29:12') MOD 1 + COS(-1)) AS c1;
+SELECT(ASIN(-1)+ LN(-1)) % (ATAN(-1) MOD FLOOR(1)) * (TRUNCATE(EXP(-1.e-2),-1.e+30) % RADIANS(-1)) * (LAST_DAY('1-03-30 1:29:12') MOD 1 + COS(-1)) AS c1;
+
+
 --echo #
 --echo # End of 10.5 tests
 --echo #

sql/item_func.cc

@@ -2638,12 +2638,12 @@ void Item_func_round::fix_arg_hex_hybrid()
 }
 
 
-double my_double_round(double value, longlong dec, bool dec_unsigned,
+double my_double_round(double value, longlong dec_value, bool dec_unsigned,
  bool truncate)
 {
  double tmp;
- bool dec_negative= (dec < 0) && !dec_unsigned;
- ulonglong abs_dec= dec_negative ? -dec : dec;
+ const Longlong_hybrid dec(dec_value, dec_unsigned);
+ const ulonglong abs_dec= dec.abs();
  /*
  tmp2 is here to avoid return the value with 80 bit precision
  This will fix that the test round(0.1,1) = round(0.1,1) is true
@@ -2658,22 +2658,24 @@ double my_double_round(double value, longlong dec, bool dec_unsigned,
  volatile double value_div_tmp= value / tmp;
  volatile double value_mul_tmp= value * tmp;
 
- if (!dec_negative && std::isinf(tmp)) // "dec" is too large positive number
+ if (!dec.neg() && std::isinf(tmp)) // "dec" is a too large positive number
  return value;
 
- if (dec_negative && std::isinf(tmp))
- tmp2= 0.0;
- else if (!dec_negative && std::isinf(value_mul_tmp))
+ if (dec.neg() && std::isinf(tmp)) // "dec" is a too small negative number
+ return 0.0;
+
+ // Handle "dec" with a reasonably small absolute value
+ if (!dec.neg() && std::isinf(value_mul_tmp))
  tmp2= value;
  else if (truncate)
  {
  if (value >= 0.0)
- tmp2= dec < 0 ? floor(value_div_tmp) * tmp : floor(value_mul_tmp) / tmp;
+ tmp2= dec.neg() ? floor(value_div_tmp) * tmp : floor(value_mul_tmp) / tmp;
  else
- tmp2= dec < 0 ? ceil(value_div_tmp) * tmp : ceil(value_mul_tmp) / tmp;
+ tmp2= dec.neg() ? ceil(value_div_tmp) * tmp : ceil(value_mul_tmp) / tmp;
  }
  else
- tmp2=dec < 0 ? rint(value_div_tmp) * tmp : rint(value_mul_tmp) / tmp;
+ tmp2=dec.neg() ? rint(value_div_tmp) * tmp : rint(value_mul_tmp) / tmp;
 
  return tmp2;
 }