Merge 10.2 into 10.3

Author: dr-m

client/mysqltest.cc

@@ -587,9 +587,10 @@ ATTRIBUTE_NORETURN
 static void cleanup_and_exit(int exit_code);
 
 ATTRIBUTE_NORETURN
-void really_die(const char *msg);
+static void really_die(const char *msg);
 void report_or_die(const char *fmt, ...);
-void die(const char *fmt, ...);
+ATTRIBUTE_NORETURN
+static void die(const char *fmt, ...);
 static void make_error_message(char *buf, size_t len, const char *fmt, va_list args);
 ATTRIBUTE_NORETURN ATTRIBUTE_FORMAT(printf, 1, 2)
 void abort_not_supported_test(const char *fmt, ...);
@@ -1555,7 +1556,7 @@ static void make_error_message(char *buf, size_t len, const char *fmt, va_list a
  s+= my_snprintf(s, end -s, "\n");
 }
 
-void die(const char *fmt, ...)
+static void die(const char *fmt, ...)
 {
  char buff[DIE_BUFF_SIZE];
  va_list args;
@@ -1564,7 +1565,7 @@ void die(const char *fmt, ...)
  really_die(buff);
 }
 
-void really_die(const char *msg)
+static void really_die(const char *msg)
 {
  static int dying= 0;
  fflush(stdout);

include/my_valgrind.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2010, 2019, MariaDB Corporation.
+/* Copyright (C) 2010, 2020, MariaDB Corporation.
 
  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
@@ -32,7 +32,9 @@
 
 #if defined(HAVE_VALGRIND_MEMCHECK_H) && defined(HAVE_valgrind)
 # include <valgrind/memcheck.h>
+# define HAVE_valgrind_or_MSAN
 # define MEM_UNDEFINED(a,len) VALGRIND_MAKE_MEM_UNDEFINED(a,len)
+# define MEM_MAKE_DEFINED(a,len) VALGRIND_MAKE_MEM_DEFINED(a,len)
 # define MEM_NOACCESS(a,len) VALGRIND_MAKE_MEM_NOACCESS(a,len)
 # define MEM_CHECK_ADDRESSABLE(a,len) VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,len)
 # define MEM_CHECK_DEFINED(a,len) VALGRIND_CHECK_MEM_IS_DEFINED(a,len)
@@ -42,28 +44,42 @@
 /* How to do manual poisoning:
 https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning */
 # define MEM_UNDEFINED(a,len) ASAN_UNPOISON_MEMORY_REGION(a,len)
+# define MEM_MAKE_DEFINED(a,len) ((void) 0)
 # define MEM_NOACCESS(a,len) ASAN_POISON_MEMORY_REGION(a,len)
 # define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0)
 # define MEM_CHECK_DEFINED(a,len) ((void) 0)
 # define REDZONE_SIZE 8
+#elif __has_feature(memory_sanitizer)
+# include <sanitizer/msan_interface.h>
+# define HAVE_valgrind_or_MSAN
+# define MEM_UNDEFINED(a,len) __msan_allocated_memory(a,len)
+# define MEM_MAKE_DEFINED(a,len) __msan_unpoison(a,len)
+# define MEM_NOACCESS(a,len) ((void) 0)
+# define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0)
+# define MEM_CHECK_DEFINED(a,len) __msan_check_mem_is_initialized(a,len)
+# define REDZONE_SIZE 8
 #else
 # define MEM_UNDEFINED(a,len) ((void) (a), (void) (len))
+# define MEM_MAKE_DEFINED(a,len) ((void) 0)
 # define MEM_NOACCESS(a,len) ((void) 0)
 # define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0)
 # define MEM_CHECK_DEFINED(a,len) ((void) 0)
 # define REDZONE_SIZE 0
 #endif /* HAVE_VALGRIND_MEMCHECK_H */
 
-#if defined(TRASH_FREED_MEMORY)
-/* NOTE: Do not invoke TRASH_FILL directly! Use TRASH_ALLOC or TRASH_FREE.
-
-The MEM_UNDEFINED() call before memset() is for canceling the effect
-of any previous MEM_NOACCESS(). We must invoke MEM_UNDEFINED() after
-writing the dummy pattern, unless MEM_NOACCESS() is going to be invoked.
-On AddressSanitizer, the MEM_UNDEFINED() in TRASH_ALLOC() has no effect. */
+#ifdef TRASH_FREED_MEMORY
+/*
+ TRASH_FILL() has to call MEM_UNDEFINED() to cancel any effect of TRASH_FREE().
+ This can happen in the case one does
+ TRASH_ALLOC(A,B) ; TRASH_FREE(A,B) ; TRASH_ALLOC(A,B)
+ to reuse the same memory in an internal memory allocator like MEM_ROOT.
+ For my_malloc() and safemalloc() the extra MEM_UNDEFINED is bit of an
+ overkill.
+ TRASH_FILL() is an internal function and should not be used externally.
+*/
 #define TRASH_FILL(A,B,C) do { const size_t trash_tmp= (B); MEM_UNDEFINED(A, trash_tmp); memset(A, C, trash_tmp); } while (0)
 #else
-#define TRASH_FILL(A,B,C) while (0)
+#define TRASH_FILL(A,B,C) do { MEM_UNDEFINED((A), (B)); } while (0)
 #endif
 /** Note that some memory became allocated or uninitialized. */
 #define TRASH_ALLOC(A,B) do { TRASH_FILL(A,B,0xA5); MEM_UNDEFINED(A,B); } while(0)

include/span.h

@@ -24,11 +24,33 @@ this program; if not, write to the Free Software Foundation, Inc.,
 namespace st_
 {
 
+namespace detail
+{
+
+template <class T> struct remove_cv
+{
+ typedef T type;
+};
+template <class T> struct remove_cv<const T>
+{
+ typedef T type;
+};
+template <class T> struct remove_cv<volatile T>
+{
+ typedef T type;
+};
+template <class T> struct remove_cv<const volatile T>
+{
+ typedef T type;
+};
+
+} // namespace detail
+
 template <class ElementType> class span
 {
 public:
  typedef ElementType element_type;
- typedef ElementType value_type;
+ typedef typename detail::remove_cv<ElementType>::type value_type;
  typedef size_t size_type;
  typedef ptrdiff_t difference_type;
  typedef element_type *pointer;
@@ -38,7 +60,6 @@ template <class ElementType> class span
  typedef pointer iterator;
  typedef const_pointer const_iterator;
  typedef std::reverse_iterator<iterator> reverse_iterator;
- typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
 
  span() : data_(NULL), size_(0) {}
 
@@ -64,73 +85,72 @@ template <class ElementType> class span
 
  span &operator=(const span &other)
  {
- data_= other.data_;
- size_= other.size_;
+ data_= other.data();
+ size_= other.size();
  return *this;
  }
 
  template <size_t Count> span<element_type> first() const
  {
  assert(!empty());
- return span(data_, 1);
+ return span(data(), 1);
  }
  template <size_t Count> span<element_type> last() const
  {
  assert(!empty());
- return span(data_ + size() - 1, 1);
+ return span(data() + size() - 1, 1);
  }
 
  span<element_type> first(size_type count) const
  {
  assert(!empty());
- return span(data_, 1);
+ return span(data(), 1);
  }
  span<element_type> last(size_type count) const
  {
  assert(!empty());
- return span(data_ + size() - 1, 1);
+ return span(data() + size() - 1, 1);
  }
  span<element_type> subspan(size_type offset, size_type count) const
  {
  assert(!empty());
  assert(size() >= offset + count);
- return span(data_ + offset, count);
+ return span(data() + offset, count);
  }
 
  size_type size() const { return size_; }
- size_type size_bytes() const { return size_ * sizeof(ElementType); }
- bool empty() const __attribute__((warn_unused_result)) { return size_ == 0; }
+ size_type size_bytes() const { return size() * sizeof(ElementType); }
+ bool empty() const __attribute__((warn_unused_result))
+ {
+ return size() == 0;
+ }
 
  reference operator[](size_type idx) const
  {
  assert(size() > idx);
- return data_[idx];
+ return data()[idx];
  }
  reference front() const
  {
  assert(!empty());
- return data_[0];
+ return data()[0];
  }
  reference back() const
  {
  assert(!empty());
- return data_[size() - 1];
- }
- pointer data() const
- {
- assert(!empty());
- return data_;
+ return data()[size() - 1];
  }
+ pointer data() const { return data_; }
 
  iterator begin() const { return data_; }
  iterator end() const { return data_ + size_; }
  reverse_iterator rbegin() const
  {
- return std::reverse_iterator<iterator>(std::advance(end(), -1));
+ return std::reverse_iterator<iterator>(end());
  }
  reverse_iterator rend() const
  {
- return std::reverse_iterator<iterator>(std::advance(begin(), -1));
+ return std::reverse_iterator<iterator>(begin());
  }
 
 private:

libmariadb

@@ -1,6 +1,8 @@
 -- source include/have_innodb.inc
 -- source include/have_example_key_management_plugin.inc
 -- source include/not_embedded.inc
+# We can't run this test under valgrind as it 'takes forever'
+-- source include/not_valgrind.inc
 
 create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb;
 create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact;

mysql-test/suite/encryption/t/innodb_encryption_tables.test

@@ -234,48 +234,3 @@ set global debug_dbug= @saved_dbug;
 drop table t1;
 set debug_sync=reset;
 SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency;
-#
-# MDEV-18546 ASAN heap-use-after-free
-# in innobase_get_computed_value / row_purge
-#
-CREATE TABLE t1 (
-pk INT AUTO_INCREMENT,
-b BIT(15),
-v BIT(15) AS (b) VIRTUAL,
-PRIMARY KEY(pk),
-UNIQUE(v)
-) ENGINE=InnoDB;
-INSERT IGNORE INTO t1 (b) VALUES
-(NULL),(b'011'),(b'000110100'),
-(b'01101101010'),(b'01111001001011'),(NULL);
-SET GLOBAL innodb_debug_sync = "ib_clust_v_col_before_row_allocated "
- "SIGNAL before_row_allocated "
- "WAIT_FOR flush_unlock";
-SET GLOBAL innodb_debug_sync = "ib_open_after_dict_open "
- "SIGNAL purge_open "
- "WAIT_FOR select_open";
-SET @saved_dbug= @@GLOBAL.debug_dbug;
-set global debug_dbug= "d,ib_purge_virtual_index_callback";
-connect purge_waiter,localhost,root;
-SET debug_sync= "now WAIT_FOR before_row_allocated";
-connection default;
-REPLACE INTO t1 (pk, b) SELECT pk, b FROM t1;
-connection purge_waiter;
-connection default;
-disconnect purge_waiter;
-FLUSH TABLES;
-SET GLOBAL innodb_debug_sync = reset;
-SET debug_sync= "now SIGNAL flush_unlock WAIT_FOR purge_open";
-SET GLOBAL innodb_debug_sync = reset;
-SET debug_sync= "ib_open_after_dict_open SIGNAL select_open";
-SELECT * FROM t1;
-pk	b	v
-1	NULL	NULL
-2		
-3	4	4
-4	j	j
-5	K	K
-6	NULL	NULL
-DROP TABLE t1;
-SET debug_sync= reset;
-set global debug_dbug= @saved_dbug;

mysql-test/suite/gcol/r/innodb_virtual_debug_purge.result

@@ -311,67 +311,3 @@ drop table t1;
 --source include/wait_until_count_sessions.inc
 set debug_sync=reset;
 SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency;
-
---echo #
---echo # MDEV-18546 ASAN heap-use-after-free
---echo # in innobase_get_computed_value / row_purge
---echo #
-
-CREATE TABLE t1 (
- pk INT AUTO_INCREMENT,
- b BIT(15),
- v BIT(15) AS (b) VIRTUAL,
- PRIMARY KEY(pk),
- UNIQUE(v)
-) ENGINE=InnoDB;
-INSERT IGNORE INTO t1 (b) VALUES
- (NULL),(b'011'),(b'000110100'),
- (b'01101101010'),(b'01111001001011'),(NULL);
-
-SET GLOBAL innodb_debug_sync = "ib_clust_v_col_before_row_allocated "
- "SIGNAL before_row_allocated "
- "WAIT_FOR flush_unlock";
-SET GLOBAL innodb_debug_sync = "ib_open_after_dict_open "
- "SIGNAL purge_open "
- "WAIT_FOR select_open";
-
-# In 10.2 trx_undo_roll_ptr_is_insert(t_roll_ptr) condition never pass in purge,
-# so this condition is forced to pass in row_vers_old_has_index_entry
-SET @saved_dbug= @@GLOBAL.debug_dbug;
-set global debug_dbug= "d,ib_purge_virtual_index_callback";
-
-# The purge starts from REPLACE command. To avoid possible race, separate
-# connection is used.
---connect(purge_waiter,localhost,root)
---send
-SET debug_sync= "now WAIT_FOR before_row_allocated";
-
---connection default
-REPLACE INTO t1 (pk, b) SELECT pk, b FROM t1;
-
---connection purge_waiter
-# Now we will definitely catch ib_clust_v_col_before_row_allocated
---reap
---connection default
---disconnect purge_waiter
-
-# purge hangs on the sync point. table is purged, ref_count is set to 0
-FLUSH TABLES;
-
-# Avoid hang on repeating purge.
-# Reset Will be applied after first record is purged
-SET GLOBAL innodb_debug_sync = reset;
-
-SET debug_sync= "now SIGNAL flush_unlock WAIT_FOR purge_open";
-
-# Avoid hang on repeating purge
-SET GLOBAL innodb_debug_sync = reset;
-
-# select unblocks purge thread
-SET debug_sync= "ib_open_after_dict_open SIGNAL select_open";
-SELECT * FROM t1;
-
-# Cleanup
-DROP TABLE t1;
-SET debug_sync= reset;
-set global debug_dbug= @saved_dbug;

mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test

@@ -1,25 +1,16 @@
-CREATE PROCEDURE populate_t1()
-BEGIN
-DECLARE i int DEFAULT 1;
-START TRANSACTION;
-WHILE (i <= 1000000) DO
-INSERT INTO t1 VALUES (i, i, CONCAT('a', i));
-SET i = i + 1;
-END WHILE;
-COMMIT;
-END|
+set use_stat_tables='preferably';
 CREATE TABLE t1(
 class INT,
 id INT,
 title VARCHAR(100)
 ) ENGINE=InnoDB;
+insert into t1 select seq, seq, concat('a', seq) from seq_1_to_500;
 SELECT COUNT(*) FROM t1;
 COUNT(*)
-1000000
-SET GLOBAL innodb_stats_persistent_sample_pages=2000;
+500
+set @@max_heap_table_size=16384;
 ANALYZE TABLE t1;
 Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
 test.t1	analyze	status	OK
 DROP TABLE t1;
-DROP PROCEDURE populate_t1;
-SET GLOBAL innodb_stats_persistent_sample_pages=default;