MDEV-24507: Server Crash using UDF in WHERE clause of VIEW

These changes are submitted under the BSD 3-clause License. The original ticket describes a server crash when using a UDF in the WHERE clause of a view. The crash also happens when using a UDF in the WHERE clause of a SELECT that uses a sub-query in the FROM clause. When the UDF does not have a _deinit function the server crashes in udf_handler::cleanup (sql/item_func.cc:3467). When the UDF has both an _init and a _deinit function but _init does not allocate memory for initid->ptr the server crashes in udf_handler::cleanup (sql/item_func.cc:3467). When the UDF has both an _init and a _deinit function and allocates/deallocates memory for initid->ptr the server crashes in the memory deallocation of the _deinit function. The sequence of events seen are: 1. A UDF, U, is created for the query. 2. The UDF _init function is called using U->initid. 3. U is cloned for the sub-query using the [default|implicit] copy constructor, resulting in V. 4. The UDF _init function is called using V->initid. U->initid and V->initid are the same value. 5. The UDF function is called. 6. The UDF _deinit function is called using U->initid. If any memory was allocated for initid->ptr it is deallocated here. 7. udf_handler::cleanup deletes the U->buffers String array. 8. The UDF _deinit function is called using V->initid. If any memory was allocated for initid->ptr it was previously deallocated and _deinit crashes the server. 9. udf_handler::cleanup deletes the V->buffers String array. V->buffers was the same values as U->buffers which was already deallocated. The server crashes. The solution is to create a[n explicit] copy constructor for udf_handler which sets not_original to true. Later, not_original is set back to false (0) after udf_handler::fix_fields has set up a new value for initid->ptr.

Author: SeanCAdams

mysql-test/main/udf.result

@@ -607,4 +607,68 @@ drop table t1;
 DROP FUNCTION avgcost;
 DROP FUNCTION avg2;
 DROP FUNCTION myfunc_double;
+#
+# MDEV-24507: Server Crash using UDF in WHERE clause of VIEW
+#
+CREATE FUNCTION myfunc_int RETURNS INTEGER SONAME "UDF_EXAMPLE_LIB";
+create table t1(pk int primary key, a varchar(20));
+create table t2(pk int primary key, a varchar(20));
+create view v1 as select pk, a from t1 union select pk, a from t2;
+insert into t1 values (1, "One"), (3, "Three"), (5, "Five");
+insert into t2 values (2, "Dos"), (4, "Quatro"), (6, "Seis");
+select pk, myfunc_int(a) from t1;
+pk	myfunc_int(a)
+1	3
+3	5
+5	4
+select pk, myfunc_int(a) from t2;
+pk	myfunc_int(a)
+2	3
+4	6
+6	4
+select pk, myfunc_int(a) from v1;
+pk	myfunc_int(a)
+1	3
+3	5
+5	4
+2	3
+4	6
+6	4
+select pk from t1 where myfunc_int(a) > 4;
+pk
+3
+select pk from (select pk, a from t1) A where myfunc_int(A.a) > 4;
+pk
+3
+set @save_optimizer_switch = @@optimizer_switch;
+set optimizer_switch = 'derived_merge=OFF';
+select pk, myfunc_int(a) from t1;
+pk	myfunc_int(a)
+1	3
+3	5
+5	4
+select pk, myfunc_int(a) from t2;
+pk	myfunc_int(a)
+2	3
+4	6
+6	4
+select pk, myfunc_int(a) from v1;
+pk	myfunc_int(a)
+1	3
+3	5
+5	4
+2	3
+4	6
+6	4
+select pk from t1 where myfunc_int(a) > 4;
+pk
+3
+select pk from (select pk, a from t1) A where myfunc_int(A.a) > 4;
+pk
+3
+set optimizer_switch = @save_optimizer_switch;
+drop view v1;
+drop table t2;
+drop table t1;
+drop function myfunc_int;
 # End of 10.4 tests

mysql-test/main/udf.test

@@ -647,4 +647,38 @@ DROP FUNCTION avgcost;
 DROP FUNCTION avg2;
 DROP FUNCTION myfunc_double;
 
+--echo #
+--echo # MDEV-24507: Server Crash using UDF in WHERE clause of VIEW
+--echo #
+
+--replace_result $UDF_EXAMPLE_SO UDF_EXAMPLE_LIB
+eval CREATE FUNCTION myfunc_int RETURNS INTEGER SONAME "$UDF_EXAMPLE_SO";
+
+create table t1(pk int primary key, a varchar(20));
+create table t2(pk int primary key, a varchar(20));
+create view v1 as select pk, a from t1 union select pk, a from t2;
+
+insert into t1 values (1, "One"), (3, "Three"), (5, "Five");
+insert into t2 values (2, "Dos"), (4, "Quatro"), (6, "Seis");
+
+select pk, myfunc_int(a) from t1;
+select pk, myfunc_int(a) from t2;
+select pk, myfunc_int(a) from v1;
+select pk from t1 where myfunc_int(a) > 4;
+select pk from (select pk, a from t1) A where myfunc_int(A.a) > 4;
+
+set @save_optimizer_switch = @@optimizer_switch;
+set optimizer_switch = 'derived_merge=OFF';
+select pk, myfunc_int(a) from t1;
+select pk, myfunc_int(a) from t2;
+select pk, myfunc_int(a) from v1;
+select pk from t1 where myfunc_int(a) > 4;
+select pk from (select pk, a from t1) A where myfunc_int(A.a) > 4;
+
+set optimizer_switch = @save_optimizer_switch;
+drop view v1;
+drop table t2;
+drop table t1;
+drop function myfunc_int;
+
 --echo # End of 10.4 tests

sql/item_func.cc

@@ -3547,6 +3547,7 @@ udf_handler::fix_fields(THD *thd, Item_func_or_sum *func,
  initid.const_item=func->const_item_cache;
  initid.decimals=func->decimals;
  initid.ptr=0;
+ not_original=0;
  for (uint i1= 0 ; i1 < arg_count ; i1++)
  buffers[i1].set_thread_specific();
 

sql/sql_udf.h

@@ -147,6 +147,20 @@ class udf_handler :public Sql_alloc
  *null_value= (my_bool) (is_null || error);
  }
  String *val_str(String *str,String *save_str);
+
+ udf_handler(const udf_handler &orig)
+ {
+ u_d = orig.u_d;
+ buffers = orig.buffers;
+ f_args = orig.f_args;
+ initid = orig.initid;
+ num_buffer = orig.num_buffer;
+ error = orig.error;
+ is_null = orig.is_null;
+ initialized = orig.initialized;
+ args = orig.args;
+ not_original = true;
+ }
 };