MDEV-30710 Incorrect operator when comparing large unsigned integers.

When constructing a SEL_TREE, an unsigned integer greater than its signed equivalent caused an incorrect comparison operator to be chosen.

Author: mariadb-RexJohnston

mysql-test/main/type_int.result

@@ -1675,5 +1675,16 @@ SELECT DISTINCT 1 FROM t1 GROUP BY 0 >> NULL WITH ROLLUP;
 1
 DROP TABLE t1;
 #
+# MDEV-30710 Incorrect operator when comparing large unsigned integers.
+#
+create table t1(c0 tinyint unique);
+insert into t1 values (127);
+insert into t1 values (-128);
+select * from t1 where 18446744073700599371 > c0;
+c0
+-128
+127
+drop table t1;
+#
 # End of 10.5 tests
 #

mysql-test/main/type_int.test

@@ -554,6 +554,15 @@ INSERT INTO t1 VALUES (1),(2);
 SELECT DISTINCT 1 FROM t1 GROUP BY 0 >> NULL WITH ROLLUP;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-30710 Incorrect operator when comparing large unsigned integers.
+--echo #
+
+create table t1(c0 tinyint unique);
+insert into t1 values (127);
+insert into t1 values (-128);
+select * from t1 where 18446744073700599371 > c0;
+drop table t1;
 
 --echo #
 --echo # End of 10.5 tests

sql/opt_range.cc

@@ -8995,7 +8995,8 @@ SEL_ARG *Field::stored_field_make_mm_leaf_bounded_int(RANGE_OPT_PARAM *param,
  DBUG_RETURN(new (param->mem_root) SEL_ARG_IMPOSSIBLE(this));
  longlong item_val= value->val_int();
 
- if (op == SCALAR_CMP_LT && item_val > 0)
+ if (op == SCALAR_CMP_LT && ((item_val > 0)
+ || (value->unsigned_flag && (ulonglong)item_val > 0 )))
  op= SCALAR_CMP_LE; // e.g. rewrite (tinyint < 200) to (tinyint <= 127)
  else if (op == SCALAR_CMP_GT && !unsigned_field &&
  !value->unsigned_flag && item_val < 0)