MDEV-23399 fixup: Avoid crash on Mariabackup shutdown

innodb_preshutdown(): Terminate the encryption threads before the page cleaner thread can be shut down. innodb_shutdown(): Always wait for the encryption threads and page cleaner to shut down. srv_shutdown_all_bg_threads(): Wait for the encryption threads and the page cleaner to shut down. (After an aborted startup, innodb_shutdown() would not be called.) row_get_background_drop_list_len_low(): Remove. os_thread_count: Remove. Alternatively, at the end of srv_shutdown_all_bg_threads() we could try to wait longer for the count to reach 0. On some platforms, an assertion os_thread_count==0 could fail even after a small delay, even though in the core dump all threads would have exited. srv_shutdown_threads(): Renamed from srv_shutdown_all_bg_threads(). Do not wait for the page cleaner to shut down, because the later innodb_shutdown(), which may invoke logs_empty_and_mark_files_at_shutdown(), assumes that it exists.

Author: dr-m

extra/mariabackup/xtrabackup.cc

@@ -3418,15 +3418,11 @@ xb_data_files_init()
 return(xb_load_tablespaces());
 }
 
-/************************************************************************
-Destroy the tablespace memory cache. */
-static
-void
-xb_data_files_close()
+/** Destroy the tablespace memory cache. */
+static void xb_data_files_close()
 {
-ut_ad(!os_thread_count);
-fil_close_all_files();
-buf_dblwr.close();
+ fil_close_all_files();
+ buf_dblwr.close();
 }
 
 /***********************************************************************

storage/innobase/buf/buf0flu.cc

@@ -1987,7 +1987,10 @@ static os_thread_ret_t DECLARE_THREAD(buf_flush_page_cleaner)(void*)
 buf_flush_wait_batch_end_acquiring_mutex(false);
 }
 
+mysql_mutex_lock(&buf_pool.flush_list_mutex);
 buf_page_cleaner_is_active = false;
+mysql_cond_broadcast(&buf_pool.done_flush_list);
+mysql_mutex_unlock(&buf_pool.flush_list_mutex);
 
 my_thread_end();
 /* We count the number of threads in os_thread_exit(). A created

storage/innobase/include/os0thread.h

@@ -66,9 +66,6 @@ typedef void* (*os_posix_f_t) (void*);
 typedef unsigned int mysql_pfs_key_t;
 #endif /* HAVE_PSI_INTERFACE */
 
-/** Number of threads active. */
-externAtomic_counter<ulint>os_thread_count;
-
 /***************************************************************//**
 Compares two thread ids for equality.
 @return TRUE if equal */

storage/innobase/os/os0thread.cc

@@ -27,9 +27,6 @@ Created 9/8/1995 Heikki Tuuri
 #include "univ.i"
 #include "srv0srv.h"
 
-/** Number of threads active. */
-Atomic_counter<ulint>	os_thread_count;
-
 /***************************************************************//**
 Compares two thread ids for equality.
 @return TRUE if equal */
@@ -110,8 +107,6 @@ os_thread_t os_thread_create(os_thread_func_t func, void *arg)
 
 CloseHandle(handle);
 
-os_thread_count++;
-
 return((os_thread_t)new_thread_id);
 #else /* _WIN32 else */
 
@@ -125,8 +120,6 @@ os_thread_t os_thread_create(os_thread_func_t func, void *arg)
 abort();
 }
 
-os_thread_count++;
-
 ret = pthread_create(&new_thread_id, &attr, func, arg);
 
 ut_a(ret == 0);
@@ -135,8 +128,6 @@ os_thread_t os_thread_create(os_thread_func_t func, void *arg)
 
 #endif /* not _WIN32 */
 
-ut_a(os_thread_count <= srv_max_n_threads);
-
 return((os_thread_t)new_thread_id);
 }
 
@@ -152,8 +143,6 @@ ATTRIBUTE_NORETURN void os_thread_exit()
 pfs_delete_thread();
 #endif
 
-os_thread_count--;
-
 #ifdef _WIN32
 ExitThread(0);
 #else

storage/innobase/srv/srv0start.cc

@@ -822,11 +822,8 @@ srv_open_tmp_tablespace(bool create_new_db)
 return(err);
 }
 
-/**
-Shutdown all background threads created by InnoDB. */
-static
-void
-srv_shutdown_all_bg_threads()
+/** Shutdown background threads, except the page cleaner. */
+static void srv_shutdown_threads()
 {
 ut_ad(!srv_undo_sources);
 srv_shutdown_state = SRV_SHUTDOWN_EXIT_THREADS;
@@ -838,38 +835,9 @@ srv_shutdown_all_bg_threads()
 srv_purge_shutdown();
 }
 
-/* All threads end up waiting for certain events. Put those events
-to the signaled state. Then the threads will exit themselves after
-os_event_wait(). */
-for (uint i = 0; i < 1000; ++i) {
-/* NOTE: IF YOU CREATE THREADS IN INNODB, YOU MUST EXIT THEM
-HERE OR EARLIER */
-
-if (!srv_read_only_mode) {
-/* b. srv error monitor thread exits automatically,
-no need to do anything here */
-if (srv_n_fil_crypt_threads_started) {
-os_event_set(fil_crypt_threads_event);
-}
-}
-
-if (buf_page_cleaner_is_active) {
-ut_ad(!srv_read_only_mode);
-
-/* e. Exit the buf_flush_page_cleaner */
-mysql_cond_signal(&buf_pool.do_flush_list);
-}
-
-if (!os_thread_count) {
-return;
-}
-
-os_thread_sleep(100000);
+if (srv_n_fil_crypt_threads) {
+fil_crypt_set_thread_cnt(0);
 }
-
-ib::warn() << os_thread_count << " threads created by InnoDB"
-" had not exited at shutdown!";
-ut_ad(0);
 }
 
 #ifdef UNIV_DEBUG
@@ -916,7 +884,7 @@ srv_init_abort_low(
 }
 
 srv_shutdown_bg_undo_sources();
-srv_shutdown_all_bg_threads();
+srv_shutdown_threads();
 return(err);
 }
 
@@ -1972,9 +1940,10 @@ dberr_t srv_start(bool create_new_db)
 /** Shut down background threads that can generate undo log. */
 void srv_shutdown_bg_undo_sources()
 {
+srv_shutdown_state = SRV_SHUTDOWN_INITIATED;
+
 if (srv_undo_sources) {
 ut_ad(!srv_read_only_mode);
-srv_shutdown_state = SRV_SHUTDOWN_INITIATED;
 fts_optimize_shutdown();
 dict_stats_shutdown();
 while (row_get_background_drop_list_len_low()) {
@@ -2010,6 +1979,9 @@ void innodb_preshutdown()
  }
  srv_shutdown_bg_undo_sources();
  srv_purge_shutdown();
+
+ if (srv_n_fil_crypt_threads)
+ fil_crypt_set_thread_cnt(0);
 }
 
 
@@ -2020,9 +1992,21 @@ void innodb_shutdown()
 ut_ad(!srv_undo_sources);
 switch (srv_operation) {
 case SRV_OPERATION_BACKUP:
-case SRV_OPERATION_RESTORE:
 case SRV_OPERATION_RESTORE_DELTA:
+break;
+case SRV_OPERATION_RESTORE:
 case SRV_OPERATION_RESTORE_EXPORT:
+srv_shutdown_state = SRV_SHUTDOWN_CLEANUP;
+if (!buf_page_cleaner_is_active) {
+break;
+}
+mysql_mutex_lock(&buf_pool.flush_list_mutex);
+while (buf_page_cleaner_is_active) {
+mysql_cond_signal(&buf_pool.do_flush_list);
+mysql_cond_wait(&buf_pool.done_flush_list,
+&buf_pool.flush_list_mutex);
+}
+mysql_mutex_unlock(&buf_pool.flush_list_mutex);
 break;
 case SRV_OPERATION_NORMAL:
 /* Shut down the persistent files. */
@@ -2032,7 +2016,8 @@ void innodb_shutdown()
 os_aio_free();
 fil_close_all_files();
 /* Exit any remaining threads. */
-srv_shutdown_all_bg_threads();
+ut_ad(!buf_page_cleaner_is_active);
+srv_shutdown_threads();
 
 if (srv_monitor_file) {
 my_fclose(srv_monitor_file, MYF(MY_WME));