MDEV-33896 : Galera test failure on galera_3nodes.MDEV-29171

Based on logs we might start SST before donor has reached Primary state. Because this test shutdowns all nodes we need to make sure when we start nodes that previous nodes have reached Primary state and joined the cluster. Signed-off-by: Julius Goryavsky <julius.goryavsky@mariadb.com>

Author: janlindstrom

mysql-test/suite/galera_3nodes/disabled.def

@@ -16,5 +16,4 @@ galera_ssl_reload : MDEV-32778 galera_ssl_reload failed with warning message
 galera_ipv6_mariabackup : temporarily disabled at the request of Codership
 galera_pc_bootstrap : temporarily disabled at the request of Codership
 galera_ipv6_mariabackup_section : temporarily disabled at the request of Codership
-MDEV-29171 : MDEV-33842 galera_3nodes.MDEV-29171 fails on shutdown_server
 GCF-354 : MDEV-25614 Galera test failure on GCF-354

mysql-test/suite/galera_3nodes/r/MDEV-29171.result

@@ -19,20 +19,40 @@ connection node_2;
 connection node_1;
 connection node_1;
 # restart: --wsrep_new_cluster --wsrep_gtid_domain_id=200
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
+VARIABLE_VALUE
+Primary
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment';
+VARIABLE_VALUE
+Synced
 show variables like 'wsrep_gtid_domain_id';
 Variable_name	Value
 wsrep_gtid_domain_id	200
 connection node_2;
 # restart
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
+VARIABLE_VALUE
+Primary
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment';
+VARIABLE_VALUE
+Synced
 show variables like 'wsrep_gtid_domain_id';
 Variable_name	Value
 wsrep_gtid_domain_id	200
+connection node_1;
 connection node_3;
 # restart: --wsrep_sst_donor=node2
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
+VARIABLE_VALUE
+Primary
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment';
+VARIABLE_VALUE
+Synced
 show variables like 'wsrep_gtid_domain_id';
 Variable_name	Value
 wsrep_gtid_domain_id	200
 connection node_1;
+connection node_1;
 set global wsrep_gtid_domain_id=100;
 connection node_2;
 set global wsrep_gtid_domain_id=100;

mysql-test/suite/galera_3nodes/t/MDEV-29171.test

@@ -50,6 +50,16 @@ select @@wsrep_gtid_domain_id,@@wsrep_node_name;
 --connection node_1
 --let $restart_parameters = --wsrep_new_cluster --wsrep_gtid_domain_id=200
 --source include/start_mysqld.inc
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready';
+--source include/wait_condition.inc
+--let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
+--source include/wait_condition.inc
+--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'
+--source include/wait_condition.inc
+
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment';
 show variables like 'wsrep_gtid_domain_id';
 
 #
@@ -59,8 +69,21 @@ show variables like 'wsrep_gtid_domain_id';
 --let $restart_parameters =
 --let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 --source include/start_mysqld.inc
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready';
+--source include/wait_condition.inc
+--let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
+--source include/wait_condition.inc
+--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'
+--source include/wait_condition.inc
+
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment';
 show variables like 'wsrep_gtid_domain_id';
 
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
 
 #
 # Restart node_3, select node_2 as donor
@@ -70,9 +93,23 @@ show variables like 'wsrep_gtid_domain_id';
 --let $restart_parameters = --wsrep_sst_donor="node2"
 --let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.3.expect
 --source include/start_mysqld.inc
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready';
+--source include/wait_condition.inc
+--let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
+--source include/wait_condition.inc
+--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'
+--source include/wait_condition.inc
+
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment';
+
 # Expect domain id 200
 show variables like 'wsrep_gtid_domain_id';
 
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
 
 #
 # Cleanup