Merge branch 'merge-xtradb-5.6' into 10.0

Author: cvicentiu

storage/xtradb/btr/btr0pcur.cc

@@ -33,6 +33,40 @@ Created 2/23/1996 Heikki Tuuri
 #include "rem0cmp.h"
 #include "trx0trx.h"
 #include "srv0srv.h"
+
+/** Updates fragmentation statistics for a single page transition.
+@param[in]	page	the current page being processed
+@param[in]	page_no	page number to move to (next_page_no
+if forward_direction is true,
+prev_page_no otherwise.
+@param[in]	forward_direction	move direction: true means moving
+forward, false - backward. */
+static
+void
+btr_update_scan_stats(const page_t* page, ulint page_no, bool forward_direction)
+{
+fragmentation_stats_tstats;
+memset(&stats, 0, sizeof(stats));
+ulint	extracted_page_no = page_get_page_no(page);
+ulint	delta = forward_direction ?
+page_no - extracted_page_no :
+extracted_page_no - page_no;
+
+if (delta == 1) {
+++stats.scan_pages_contiguous;
+} else {
+++stats.scan_pages_disjointed;
+}
+stats.scan_pages_total_seek_distance +=
+extracted_page_no > page_no ?
+extracted_page_no - page_no :
+page_no - extracted_page_no;
+
+stats.scan_data_size += page_get_data_size(page);
+stats.scan_deleted_recs_size +=
+page_header_get_field(page, PAGE_GARBAGE);
+thd_add_fragmentation_stats(current_thd, &stats);
+}
 /**************************************************************//**
 Allocates memory for a persistent cursor object and initializes the cursor.
 @return	own: persistent cursor */
@@ -426,6 +460,8 @@ btr_pcur_move_to_next_page(
 
 ut_ad(next_page_no != FIL_NULL);
 
+btr_update_scan_stats(page, next_page_no, true /* forward */);
+
 next_block = btr_block_get(space, zip_size, next_page_no,
  cursor->latch_mode,
  btr_pcur_get_btr_cur(cursor)->index, mtr);
@@ -509,6 +545,10 @@ btr_pcur_move_backward_from_page(
 
 prev_page_no = btr_page_get_prev(page, mtr);
 
+if (prev_page_no != FIL_NULL) {
+btr_update_scan_stats(page, prev_page_no, false /* backward */);
+}
+
 if (prev_page_no == FIL_NULL) {
 } else if (btr_pcur_is_before_first_on_page(cursor)) {
 

storage/xtradb/buf/buf0rea.cc

@@ -123,7 +123,12 @@ buf_read_page_low(
 use to stop dangling page reads from a tablespace
 which we have DISCARDed + IMPORTed back */
 ulint	offset,/*!< in: page number */
-trx_t*	trx)
+trx_t*	trx,
+boolshould_buffer)/*!< in: whether to buffer an aio request.
+AIO read ahead uses this. If you plan to
+use this parameter, make sure you remember
+to call os_aio_dispatch_read_array_submit()
+when you're ready to commit all your requests.*/
 {
 buf_page_t*	bpage;
 ulint	wake_later;
@@ -229,14 +234,15 @@ buf_read_page_low(
 *err = _fil_io(OS_FILE_READ | wake_later
  | ignore_nonexistent_pages,
  sync, space, zip_size, offset, 0, zip_size,
- bpage->zip.data, bpage, trx);
+ bpage->zip.data, bpage, trx, should_buffer);
 } else {
 ut_a(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE);
 
 *err = _fil_io(OS_FILE_READ | wake_later
  | ignore_nonexistent_pages,
  sync, space, 0, offset, 0, UNIV_PAGE_SIZE,
- ((buf_block_t*) bpage)->frame, bpage, trx);
+ ((buf_block_t*) bpage)->frame, bpage, trx,
+ should_buffer);
 }
 
 if (sync) {
@@ -395,7 +401,7 @@ buf_read_ahead_random(
 &err, false,
 ibuf_mode | OS_AIO_SIMULATED_WAKE_LATER,
 space, zip_size, FALSE,
-tablespace_version, i, trx);
+tablespace_version, i, trx, false);
 if (err == DB_TABLESPACE_DELETED) {
 ut_print_timestamp(stderr);
 fprintf(stderr,
@@ -459,7 +465,7 @@ buf_read_page(
 
 count = buf_read_page_low(&err, true, BUF_READ_ANY_PAGE, space,
  zip_size, FALSE,
- tablespace_version, offset, trx);
+ tablespace_version, offset, trx, false);
 srv_stats.buf_pool_reads.add(count);
 if (err == DB_TABLESPACE_DELETED) {
 ut_print_timestamp(stderr);
@@ -507,7 +513,7 @@ buf_read_page_async(
  | OS_AIO_SIMULATED_WAKE_LATER
  | BUF_READ_IGNORE_NONEXISTENT_PAGES,
  space, zip_size, FALSE,
- tablespace_version, offset, NULL);
+ tablespace_version, offset, NULL, false);
 srv_stats.buf_pool_reads.add(count);
 
 /* We do not increment number of I/O operations used for LRU policy
@@ -775,7 +781,8 @@ buf_read_ahead_linear(
 count += buf_read_page_low(
 &err, false,
 ibuf_mode,
-space, zip_size, FALSE, tablespace_version, i, trx);
+space, zip_size, FALSE, tablespace_version,
+i, trx, true);
 if (err == DB_TABLESPACE_DELETED) {
 ut_print_timestamp(stderr);
 fprintf(stderr,
@@ -788,6 +795,7 @@ buf_read_ahead_linear(
 }
 }
 }
+os_aio_dispatch_read_array_submit();
 
 /* In simulated aio we wake the aio handler threads only after
 queuing all aio requests, in native aio the following call does
@@ -865,7 +873,7 @@ buf_read_ibuf_merge_pages(
 buf_read_page_low(&err, sync && (i + 1 == n_stored),
  BUF_READ_ANY_PAGE, space_ids[i],
  zip_size, TRUE, space_versions[i],
- page_nos[i], NULL);
+ page_nos[i], NULL, false);
 
 if (UNIV_UNLIKELY(err == DB_TABLESPACE_DELETED)) {
 tablespace_deleted:
@@ -1001,12 +1009,13 @@ buf_read_recv_pages(
 if ((i + 1 == n_stored) && sync) {
 buf_read_page_low(&err, true, BUF_READ_ANY_PAGE, space,
  zip_size, TRUE, tablespace_version,
- page_nos[i], NULL);
+ page_nos[i], NULL, false);
 } else {
 buf_read_page_low(&err, false, BUF_READ_ANY_PAGE
  | OS_AIO_SIMULATED_WAKE_LATER,
  space, zip_size, TRUE,
- tablespace_version, page_nos[i], NULL);
+ tablespace_version, page_nos[i],
+ NULL, false);
 }
 }
 

storage/xtradb/dict/dict0stats_bg.cc

@@ -203,7 +203,7 @@ dict_stats_wait_bg_to_stop_using_table(
 unlocking/locking the data dict */
 {
 while (!dict_stats_stop_bg(table)) {
-DICT_STATS_BG_YIELD(trx);
+DICT_BG_YIELD(trx);
 }
 }
 

storage/xtradb/fil/fil0fil.cc

@@ -5111,7 +5111,7 @@ fil_extend_space_to_desired_size(
 success = os_aio(OS_FILE_WRITE, OS_AIO_SYNC,
  node->name, node->handle, buf,
  offset, page_size * n_pages,
- NULL, NULL, space_id, NULL);
+ NULL, NULL, space_id, NULL, false);
 #endif /* UNIV_HOTBACKUP */
 
 DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28",
@@ -5485,7 +5485,12 @@ _fil_io(
 appropriately aligned */
 void*	message,/*!< in: message for aio handler if non-sync
 aio used, else ignored */
-trx_t*	trx)
+trx_t*	trx,
+boolshould_buffer)/*!< in: whether to buffer an aio request.
+AIO read ahead uses this. If you plan to
+use this parameter, make sure you remember
+to call os_aio_dispatch_read_array_submit()
+when you're ready to commit all your requests.*/
 {
 ulint	mode;
 fil_space_t*	space;
@@ -5705,7 +5710,7 @@ _fil_io(
 const char* name = node->name == NULL ? space->name : node->name;
 
 ret = os_aio(type, mode | wake_later, name, node->handle, buf,
-offset, len, node, message, space_id, trx);
+offset, len, node, message, space_id, trx, should_buffer);
 
 #else
 /* In mysqlbackup do normal i/o, not aio */

storage/xtradb/fts/fts0fts.cc

@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2011, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2016, 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -26,6 +26,7 @@ Full Text Search interface
 #include "row0mysql.h"
 #include "row0upd.h"
 #include "dict0types.h"
+#include "dict0stats_bg.h"
 #include "row0sel.h"
 
 #include "fts0fts.h"
@@ -868,18 +869,37 @@ fts_drop_index(
 
 err = fts_drop_index_tables(trx, index);
 
-fts_free(table);
-
+for(;;) {
+bool retry = false;
+if (index->index_fts_syncing) {
+retry = true;
+}
+if (!retry){
+fts_free(table);
+break;
+}
+DICT_BG_YIELD(trx);
+}
 return(err);
 }
 
-current_doc_id = table->fts->cache->next_doc_id;
-first_doc_id = table->fts->cache->first_doc_id;
-fts_cache_clear(table->fts->cache);
-fts_cache_destroy(table->fts->cache);
-table->fts->cache = fts_cache_create(table);
-table->fts->cache->next_doc_id = current_doc_id;
-table->fts->cache->first_doc_id = first_doc_id;
+for(;;) {
+bool retry = false;
+if (index->index_fts_syncing) {
+retry = true;
+}
+if (!retry){
+current_doc_id = table->fts->cache->next_doc_id;
+first_doc_id = table->fts->cache->first_doc_id;
+fts_cache_clear(table->fts->cache);
+fts_cache_destroy(table->fts->cache);
+table->fts->cache = fts_cache_create(table);
+table->fts->cache->next_doc_id = current_doc_id;
+table->fts->cache->first_doc_id = first_doc_id;
+break;
+}
+DICT_BG_YIELD(trx);
+}
 } else {
 fts_cache_t* cache = table->fts->cache;
 fts_index_cache_t* index_cache;
@@ -889,9 +909,17 @@ fts_drop_index(
 index_cache = fts_find_index_cache(cache, index);
 
 if (index_cache != NULL) {
-if (index_cache->words) {
-fts_words_free(index_cache->words);
-rbt_free(index_cache->words);
+for(;;) {
+bool retry = false;
+if (index->index_fts_syncing) {
+retry = true;
+}
+if (!retry && index_cache->words) {
+fts_words_free(index_cache->words);
+rbt_free(index_cache->words);
+break;
+}
+DICT_BG_YIELD(trx);
 }
 
 ib_vector_remove(cache->indexes, *(void**) index_cache);
@@ -4568,10 +4596,16 @@ fts_sync(
 index_cache = static_cast<fts_index_cache_t*>(
 ib_vector_get(cache->indexes, i));
 
-if (index_cache->index->to_be_dropped) {
+if (index_cache->index->to_be_dropped
+ || index_cache->index->table->to_be_dropped) {
 continue;
 }
 
+index_cache->index->index_fts_syncing = true;
+DBUG_EXECUTE_IF("fts_instrument_sync_sleep_drop_waits",
+os_thread_sleep(10000000);
+);
+
 error = fts_sync_index(sync, index_cache);
 
 if (error != DB_SUCCESS && !sync->interrupted) {
@@ -4604,11 +4638,33 @@ fts_sync(
 end_sync:
 if (error == DB_SUCCESS && !sync->interrupted) {
 error = fts_sync_commit(sync);
+if (error == DB_SUCCESS) {
+for (i = 0; i < ib_vector_size(cache->indexes); ++i) {
+fts_index_cache_t* index_cache;
+index_cache = static_cast<fts_index_cache_t*>(
+ib_vector_get(cache->indexes, i));
+if (index_cache->index->index_fts_syncing) {
+index_cache->index->index_fts_syncing
+= false;
+}
+}
+}
 } else {
 fts_sync_rollback(sync);
 }
 
 rw_lock_x_lock(&cache->lock);
+/* Clear fts syncing flags of any indexes incase sync is
+interrupeted */
+for (i = 0; i < ib_vector_size(cache->indexes); ++i) {
+fts_index_cache_t* index_cache;
+index_cache = static_cast<fts_index_cache_t*>(
+ ib_vector_get(cache->indexes, i));
+if (index_cache->index->index_fts_syncing == true) {
+index_cache->index->index_fts_syncing = false;
+ }
+}
+
 sync->interrupted = false;
 sync->in_progress = false;
 os_event_set(sync->event);

storage/xtradb/fts/fts0opt.cc

@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2007, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2016, MariaDB Corporation. All Rights reserved.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -2971,13 +2971,6 @@ fts_optimize_sync_table(
 {
 dict_table_t* table = NULL;
 
-/* Prevent DROP INDEX etc. from running when we are syncing
-cache in background. */
-if (!rw_lock_s_lock_nowait(&dict_operation_lock, __FILE__, __LINE__)) {
-/* Exit when fail to get dict operation lock. */
-return;
-}
-
 table = dict_table_open_on_id(table_id, FALSE, DICT_TABLE_OP_NORMAL);
 
 if (table) {
@@ -2987,8 +2980,6 @@ fts_optimize_sync_table(
 
 dict_table_close(table, FALSE, FALSE);
 }
-
-rw_lock_s_unlock(&dict_operation_lock);
 }
 
 /**********************************************************************//**

storage/xtradb/fts/fts0que.cc

@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2007, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -3656,6 +3656,7 @@ fts_query_free(
 
 if (query->intersection) {
 fts_query_free_doc_ids(query, query->intersection);
+query->intersection = NULL;
 }
 
 if (query->doc_ids) {