0

On an computer with 64G of ram I have a mariadb server that uses all available ram even though it is (i think at least) configured to use much less:

SHOW VARIABLES aria_block_size 8192 aria_checkpoint_interval 30 aria_checkpoint_log_activity 1048576 aria_encrypt_tables OFF aria_force_start_after_recovery_failures 0 aria_group_commit none aria_group_commit_interval 0 aria_log_file_size 1073741824 aria_log_purge_type immediate aria_max_sort_file_size 9223372036853727232 aria_page_checksum ON aria_pagecache_age_threshold 300 aria_pagecache_buffer_size 134217728 aria_pagecache_division_limit 100 aria_pagecache_file_hash_size 512 aria_recover NORMAL aria_repair_threads 1 aria_sort_buffer_size 268434432 aria_stats_method nulls_unequal aria_sync_log_dir NEWFILE aria_used_for_temp_tables ON auto_increment_increment 1 auto_increment_offset 1 autocommit ON automatic_sp_privileges ON back_log 152 basedir /usr big_tables OFF binlog_annotate_row_events OFF binlog_cache_size 32768 binlog_checksum NONE binlog_commit_wait_count 0 binlog_commit_wait_usec 100000 binlog_direct_non_transactional_updates OFF binlog_format STATEMENT binlog_optimize_thread_scheduling ON binlog_row_image FULL binlog_stmt_cache_size 32768 bulk_insert_buffer_size 8388608 character_set_client utf8mb4 character_set_connection utf8mb4 character_set_database utf8mb4 character_set_filesystem binary character_set_results utf8mb4 character_set_server utf8mb4 character_set_system utf8 character_sets_dir /usr/share/mysql/charsets/ collation_connection utf8mb4_unicode_ci collation_database utf8mb4_general_ci collation_server utf8mb4_general_ci completion_type NO_CHAIN concurrent_insert AUTO connect_timeout 10 core_file OFF datadir /var/lib/mysql/ date_format %Y-%m-%d datetime_format %Y-%m-%d %H:%i:%s deadlock_search_depth_long 15 deadlock_search_depth_short 4 deadlock_timeout_long 50000000 deadlock_timeout_short 10000 debug_no_thread_alarm OFF default_master_connection default_regex_flags default_storage_engine InnoDB default_tmp_storage_engine default_week_format 0 delay_key_write ON delayed_insert_limit 100 delayed_insert_timeout 300 delayed_queue_size 1000 div_precision_increment 4 encrypt_binlog OFF encrypt_tmp_disk_tables OFF encrypt_tmp_files OFF enforce_storage_engine error_count 0 event_scheduler OFF expensive_subquery_limit 100 expire_logs_days 10 explicit_defaults_for_timestamp OFF external_user extra_max_connections 1 extra_port 0 flush OFF flush_time 0 foreign_key_checks ON ft_boolean_syntax + -><()~*:""&| ft_max_word_len 84 ft_min_word_len 4 ft_query_expansion_limit 20 ft_stopword_file (built-in) general_log OFF general_log_file prod4.log group_concat_max_len 1024 gtid_binlog_pos gtid_binlog_state gtid_current_pos gtid_domain_id 0 gtid_ignore_duplicates OFF gtid_seq_no 0 gtid_slave_pos gtid_strict_mode OFF have_compress YES have_crypt YES have_dynamic_loading YES have_geometry YES have_openssl NO have_profiling YES have_query_cache YES have_rtree_keys YES have_ssl DISABLED have_symlink YES histogram_size 0 histogram_type SINGLE_PREC_HB host_cache_size 628 hostname prod4 identity 0 ignore_builtin_innodb OFF ignore_db_dirs in_transaction 0 init_connect init_file init_slave innodb_adaptive_flushing ON innodb_adaptive_flushing_lwm 10.000000 innodb_adaptive_hash_index ON innodb_adaptive_hash_index_partitions 1 innodb_adaptive_max_sleep_delay 150000 innodb_additional_mem_pool_size 8388608 innodb_api_bk_commit_interval 5 innodb_api_disable_rowlock OFF innodb_api_enable_binlog OFF innodb_api_enable_mdl OFF innodb_api_trx_level 0 innodb_autoextend_increment 64 innodb_autoinc_lock_mode 1 innodb_background_scrub_data_check_interval 3600 innodb_background_scrub_data_compressed OFF innodb_background_scrub_data_interval 604800 innodb_background_scrub_data_uncompressed OFF innodb_buf_dump_status_frequency 0 innodb_buffer_pool_dump_at_shutdown OFF innodb_buffer_pool_dump_now OFF innodb_buffer_pool_dump_pct 100 innodb_buffer_pool_filename ib_buffer_pool innodb_buffer_pool_instances 8 innodb_buffer_pool_load_abort OFF innodb_buffer_pool_load_at_startup OFF innodb_buffer_pool_load_now OFF innodb_buffer_pool_populate OFF innodb_buffer_pool_size 12884901888 innodb_change_buffer_max_size 25 innodb_change_buffering all innodb_checksum_algorithm INNODB innodb_checksums ON innodb_cleaner_lsn_age_factor HIGH_CHECKPOINT innodb_cmp_per_index_enabled OFF innodb_commit_concurrency 0 innodb_compression_algorithm zlib innodb_compression_failure_threshold_pct 5 innodb_compression_level 6 innodb_compression_pad_pct_max 50 innodb_concurrency_tickets 5000 innodb_corrupt_table_action assert innodb_data_file_path ibdata1:12M:autoextend innodb_data_home_dir innodb_default_encryption_key_id 1 innodb_default_row_format compact innodb_defragment OFF innodb_defragment_fill_factor 0.900000 innodb_defragment_fill_factor_n_recs 20 innodb_defragment_frequency 40 innodb_defragment_n_pages 7 innodb_defragment_stats_accuracy 0 innodb_disable_sort_file_cache OFF innodb_disallow_writes OFF innodb_doublewrite ON innodb_empty_free_list_algorithm BACKOFF innodb_encrypt_log OFF innodb_encrypt_tables OFF innodb_encryption_rotate_key_age 1 innodb_encryption_rotation_iops 100 innodb_encryption_threads 0 innodb_fake_changes OFF innodb_fast_shutdown 1 innodb_fatal_semaphore_wait_threshold 600 innodb_file_format Antelope innodb_file_format_check ON innodb_file_format_max Antelope innodb_file_per_table ON innodb_flush_log_at_timeout 1 innodb_flush_log_at_trx_commit 0 innodb_flush_method O_DIRECT innodb_flush_neighbors 1 innodb_flushing_avg_loops 30 innodb_force_load_corrupted OFF innodb_force_primary_key OFF innodb_force_recovery 0 innodb_foreground_preflush EXPONENTIAL_BACKOFF innodb_ft_aux_table innodb_ft_cache_size 8000000 innodb_ft_enable_diag_print OFF innodb_ft_enable_stopword ON innodb_ft_max_token_size 84 innodb_ft_min_token_size 3 innodb_ft_num_word_optimize 2000 innodb_ft_result_cache_limit 2000000000 innodb_ft_server_stopword_table innodb_ft_sort_pll_degree 2 innodb_ft_total_cache_size 640000000 innodb_ft_user_stopword_table innodb_idle_flush_pct 100 innodb_immediate_scrub_data_uncompressed OFF innodb_instrument_semaphores OFF innodb_io_capacity 400 innodb_io_capacity_max 2000 innodb_kill_idle_transaction 0 innodb_large_prefix OFF innodb_lock_schedule_algorithm fcfs innodb_lock_wait_timeout 50 innodb_locking_fake_changes ON innodb_locks_unsafe_for_binlog OFF innodb_log_arch_dir ./ innodb_log_arch_expire_sec 0 innodb_log_archive OFF innodb_log_block_size 512 innodb_log_buffer_size 8388608 innodb_log_checksum_algorithm INNODB innodb_log_compressed_pages ON innodb_log_file_size 1073741824 innodb_log_files_in_group 2 innodb_log_group_home_dir ./ innodb_lru_scan_depth 1024 innodb_max_bitmap_file_size 104857600 innodb_max_changed_pages 1000000 innodb_max_dirty_pages_pct 75.000000 innodb_max_dirty_pages_pct_lwm 0.001000 innodb_max_purge_lag 0 innodb_max_purge_lag_delay 0 innodb_mirrored_log_groups 1 innodb_monitor_disable innodb_monitor_enable innodb_monitor_reset innodb_monitor_reset_all innodb_mtflush_threads 8 innodb_old_blocks_pct 37 innodb_old_blocks_time 1000 innodb_online_alter_log_max_size 134217728 innodb_open_files 400 innodb_optimize_fulltext_only OFF innodb_page_size 16384 innodb_prefix_index_cluster_optimization OFF innodb_print_all_deadlocks OFF innodb_print_lock_wait_timeout_info OFF innodb_purge_batch_size 300 innodb_purge_threads 1 innodb_random_read_ahead OFF innodb_read_ahead_threshold 56 innodb_read_io_threads 64 innodb_read_only OFF innodb_replication_delay 0 innodb_rollback_on_timeout OFF innodb_rollback_segments 128 innodb_sched_priority_cleaner 19 innodb_scrub_log OFF innodb_scrub_log_speed 256 innodb_show_locks_held 10 innodb_show_verbose_locks 0 innodb_simulate_comp_failures 0 innodb_sort_buffer_size 1048576 innodb_spin_wait_delay 6 innodb_stats_auto_recalc ON innodb_stats_include_delete_marked OFF innodb_stats_method nulls_equal innodb_stats_modified_counter 0 innodb_stats_on_metadata OFF innodb_stats_persistent ON innodb_stats_persistent_sample_pages 20 innodb_stats_sample_pages 8 innodb_stats_traditional ON innodb_stats_transient_sample_pages 8 innodb_status_output OFF innodb_status_output_locks OFF innodb_strict_mode OFF innodb_support_xa ON innodb_sync_array_size 1 innodb_sync_spin_loops 30 innodb_table_locks ON innodb_thread_concurrency 0 innodb_thread_sleep_delay 10000 innodb_tmpdir innodb_track_changed_pages OFF innodb_undo_directory . innodb_undo_logs 128 innodb_undo_tablespaces 0 innodb_use_atomic_writes OFF innodb_use_fallocate OFF innodb_use_global_flush_log_at_trx_commit ON innodb_use_mtflush OFF innodb_use_native_aio ON innodb_use_stacktrace OFF innodb_use_sys_malloc ON innodb_use_trim OFF innodb_version 5.6.42-84.2 innodb_write_io_threads 4 insert_id 0 interactive_timeout 28800 join_buffer_size 262144 join_buffer_space_limit 2097152 join_cache_level 2 keep_files_on_create OFF key_buffer_size 33554432 key_cache_age_threshold 300 key_cache_block_size 1024 key_cache_division_limit 100 key_cache_file_hash_size 512 key_cache_segments 0 large_files_support ON large_page_size 0 large_pages OFF last_gtid last_insert_id 0 lc_messages en_US lc_messages_dir /usr/share/mysql lc_time_names en_US license GPL local_infile ON lock_wait_timeout 31536000 locked_in_memory OFF log_bin OFF log_bin_basename log_bin_index log_bin_trust_function_creators OFF log_error /var/log/mysql/error.log log_output FILE log_queries_not_using_indexes OFF log_slave_updates OFF log_slow_admin_statements OFF log_slow_filter admin,filesort,filesort_on_disk,full_join,full_sca... log_slow_rate_limit 1 log_slow_slave_statements OFF log_slow_verbosity log_tc_size 24576 log_warnings 1 long_query_time 10.000000 low_priority_updates OFF lower_case_file_system OFF lower_case_table_names 0 master_verify_checksum OFF max_allowed_packet 16777216 max_binlog_cache_size 18446744073709547520 max_binlog_size 104857600 max_binlog_stmt_cache_size 18446744073709547520 max_connect_errors 100 max_connections 512 max_delayed_threads 20 max_digest_length 1024 max_error_count 64 max_heap_table_size 16777216 max_insert_delayed_threads 20 max_join_size 18446744073709551615 max_length_for_sort_data 1024 max_long_data_size 16777216 max_prepared_stmt_count 16382 max_relay_log_size 104857600 max_seeks_for_key 4294967295 max_session_mem_used 9223372036854775807 max_sort_length 1024 max_sp_recursion_depth 0 max_statement_time 0.000000 max_tmp_tables 32 max_user_connections 0 max_write_lock_count 4294967295 metadata_locks_cache_size 1024 metadata_locks_hash_instances 8 min_examined_row_limit 0 mrr_buffer_size 262144 multi_range_count 256 myisam_block_size 1024 myisam_data_pointer_size 6 myisam_max_sort_file_size 9223372036853727232 myisam_mmap_size 18446744073709551615 myisam_recover_options BACKUP myisam_repair_threads 1 myisam_sort_buffer_size 134216704 myisam_stats_method NULLS_UNEQUAL myisam_use_mmap OFF mysql56_temporal_format ON net_buffer_length 16384 net_read_timeout 30 net_retry_count 10 net_write_timeout 60 old OFF old_alter_table OFF old_mode old_passwords OFF open_files_limit 8551 optimizer_prune_level 1 optimizer_search_depth 62 optimizer_selectivity_sampling_limit 100 optimizer_switch index_merge=on,index_merge_union=on,index_merge_so... optimizer_use_condition_selectivity 1 performance_schema ON performance_schema_accounts_size 100 performance_schema_digests_size 10000 performance_schema_events_stages_history_long_size 10000 performance_schema_events_stages_history_size 10 performance_schema_events_statements_history_long_... 10000 performance_schema_events_statements_history_size 10 performance_schema_events_waits_history_long_size 10000 performance_schema_events_waits_history_size 10 performance_schema_hosts_size 100 performance_schema_max_cond_classes 80 performance_schema_max_cond_instances 3348 performance_schema_max_digest_length 1024 performance_schema_max_file_classes 50 performance_schema_max_file_handles 32768 performance_schema_max_file_instances 3077 performance_schema_max_mutex_classes 200 performance_schema_max_mutex_instances 10072 performance_schema_max_rwlock_classes 40 performance_schema_max_rwlock_instances 5024 performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1044 performance_schema_max_stage_classes 150 performance_schema_max_statement_classes 178 performance_schema_max_table_handles 4000 performance_schema_max_table_instances 12500 performance_schema_max_thread_classes 50 performance_schema_max_thread_instances 1124 performance_schema_session_connect_attrs_size 512 performance_schema_setup_actors_size 100 performance_schema_setup_objects_size 100 performance_schema_users_size 100 pid_file /var/run/mysqld/mysqld.pid plugin_dir /usr/lib/x86_64-linux-gnu/mariadb18/plugin/ plugin_maturity unknown port 3306 preload_buffer_size 32768 profiling OFF profiling_history_size 15 progress_report_time 5 protocol_version 10 proxy_user pseudo_slave_mode OFF pseudo_thread_id 16370 query_alloc_block_size 16384 query_cache_limit 16777216 query_cache_min_res_unit 4096 query_cache_size 0 query_cache_strip_comments OFF query_cache_type OFF query_cache_wlock_invalidate OFF query_prealloc_size 24576 rand_seed1 357956615 rand_seed2 975103936 range_alloc_block_size 4096 read_buffer_size 131072 read_only OFF read_rnd_buffer_size 262144 relay_log relay_log_basename relay_log_index relay_log_info_file relay-log.info relay_log_purge ON relay_log_recovery OFF relay_log_space_limit 0 replicate_annotate_row_events OFF replicate_do_db replicate_do_table replicate_events_marked_for_skip REPLICATE replicate_ignore_db replicate_ignore_table replicate_wild_do_table replicate_wild_ignore_table report_host report_password report_port 3306 report_user rowid_merge_buff_size 8388608 secure_auth ON secure_file_priv server_id 0 skip_external_locking ON skip_name_resolve OFF skip_networking OFF skip_parallel_replication OFF skip_replication OFF skip_show_database OFF slave_compressed_protocol OFF slave_ddl_exec_mode IDEMPOTENT slave_domain_parallel_threads 0 slave_exec_mode STRICT slave_load_tmpdir /tmp slave_max_allowed_packet 1073741824 slave_net_timeout 3600 slave_parallel_max_queued 131072 slave_parallel_mode conservative slave_parallel_threads 0 slave_run_triggers_for_rbr NO slave_skip_errors OFF slave_sql_verify_checksum ON slave_transaction_retries 10 slave_type_conversions slow_launch_time 2 slow_query_log OFF slow_query_log_file prod4-slow.log socket /var/run/mysqld/mysqld.sock sort_buffer_size 2097152 sql_auto_is_null OFF sql_big_selects ON sql_buffer_result OFF sql_log_bin ON sql_log_off OFF sql_mode NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION sql_notes ON sql_quote_show_create ON sql_safe_updates OFF sql_select_limit 18446744073709551615 sql_slave_skip_counter 0 sql_warnings OFF ssl_ca ssl_capath ssl_cert ssl_cipher ssl_crl ssl_crlpath ssl_key storage_engine InnoDB stored_program_cache 256 strict_password_validation ON sync_binlog 0 sync_frm ON sync_master_info 10000 sync_relay_log 10000 sync_relay_log_info 10000 system_time_zone UTC table_definition_cache 400 table_open_cache 4000 thread_cache_size 8 thread_concurrency 10 thread_handling one-thread-per-connection thread_pool_idle_timeout 60 thread_pool_max_threads 1000 thread_pool_oversubscribe 3 thread_pool_size 8 thread_pool_stall_limit 500 thread_stack 196608 time_format %H:%i:%s time_zone SYSTEM timed_mutexes OFF timestamp 1561288673.757972 tmp_table_size 16777216 tmpdir /tmp transaction_alloc_block_size 8192 transaction_prealloc_size 4096 tx_isolation REPEATABLE-READ tx_read_only OFF unique_checks ON updatable_views_with_limit YES use_stat_tables NEVER userstat OFF version 10.1.38-MariaDB-0ubuntu0.18.10.2 version_comment Ubuntu 18.10 version_compile_machine x86_64 version_compile_os debian-linux-gnu version_malloc_library system jemalloc version_ssl_library YaSSL 2.4.4 wait_timeout 28800 warning_count 0 wsrep_osu_method TOI wsrep_auto_increment_control ON wsrep_causal_reads OFF wsrep_certification_rules strict wsrep_certify_nonpk ON wsrep_cluster_address wsrep_cluster_name my_wsrep_cluster wsrep_convert_lock_to_trx OFF wsrep_data_home_dir /var/lib/mysql/ wsrep_dbug_option wsrep_debug OFF wsrep_desync OFF wsrep_dirty_reads OFF wsrep_drupal_282555_workaround OFF wsrep_forced_binlog_format NONE wsrep_gtid_domain_id 0 wsrep_gtid_mode OFF wsrep_load_data_splitting ON wsrep_log_conflicts OFF wsrep_max_ws_rows 0 wsrep_max_ws_size 2147483647 wsrep_mysql_replication_bundle 0 wsrep_node_address wsrep_node_incoming_address AUTO wsrep_node_name prod4 wsrep_notify_cmd wsrep_on OFF wsrep_patch_version wsrep_25.24 wsrep_provider none wsrep_provider_options wsrep_recover OFF wsrep_reject_queries NONE wsrep_replicate_myisam OFF wsrep_restart_slave OFF wsrep_retry_autocommit 1 wsrep_slave_fk_checks ON wsrep_slave_uk_checks OFF wsrep_slave_threads 1 wsrep_sst_auth wsrep_sst_donor wsrep_sst_donor_rejects_queries OFF wsrep_sst_method rsync wsrep_sst_receive_address AUTO wsrep_start_position 00000000-0000-0000-0000-000000000000:-1 wsrep_sync_wait 0

Output of tuning-primer also seems fine:

INNODB STATUS Current InnoDB index space = 42.05 G Current InnoDB data space = 48.17 G Current InnoDB buffer pool free = 40 % Current innodb_buffer_pool_size = 12.00 G Depending on how much space your innodb indexes take up it may be safe to increase this value to up to 2 / 3 of total system memory MEMORY USAGE Max Memory Ever Allocated : 12.16 G Configured Max Per-thread Buffers : 1.40 G Configured Max Global Buffers : 12.04 G Configured Max Memory Limit : 13.45 G Physical Memory : 62.79 G Max memory limit seem to be within acceptable norms

After a few hours the output of top look like this:

 PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 26350 mysql 20 0 64.3g 55.3g 19300 S 29.9 88.0 354:20.10 /usr/sbin/mysqld

shortly after this oom_reaper would kill mysql:

[3308788.693609] Out of memory: Kill process 30421 (mysqld) score 915 or sacrifice child [3308788.693727] Killed process 30421 (mysqld) total-vm:78894468kB, anon-rss:64508740kB, file-rss:0kB, shmem-rss:0kB [3308790.493095] oom_reaper: reaped process 30421 (mysqld), now anon-rss:0kB, file-rss:0kB, shmem-rss:0kB

Im using 10.1.38-MariaDB-0ubuntu0.18.10.2 on ubuntu 18.10

Pretty much all of the table are innodb, result from show global status is here: https://pastebin.com/7ayJBpgC

New settings, after changing some settings as Rick James suggested: https://pastebin.com/N55AzWFw

Improve this question
3
  • Nothing obvious. Could you also provide SHOW GLOBAL STATUS. What Engine is being used by your tables? Commented Jun 23, 2019 at 15:25
  • Any idea of what queries were run shortly before the OOM? Commented Jun 23, 2019 at 15:29
  • I have added result of show global status (while mysql is consuming ~48g of ram) to the original post, not sure which queries are running when it gets killed, but since all the queries finish pretty fast and the usage ramps slowly (it takes ~12 hours to fill all available ram) it probably doesn't matter? Commented Jun 23, 2019 at 21:40

3 Answers 3

Reset to default
1

Analysis of GLOBAL STATUS and VARIABLES. Alas, nothing clearly points to why you are having that Memory problem.

Observations:

  • Version: 10.1.38-MariaDB-0ubuntu0.18.10.2
  • 64 GB of RAM
  • Uptime = 10:28:11; some GLOBAL STATUS values may not be meaningful yet.
  • You are not running on Windows.
  • Running 64-bit version
  • You appear to be running entirely (or mostly) InnoDB.

The More Important Issues:

thread_cache_size -- Increase to 50.

long_query_time = 1 and turn on the slowlog. More discussion: http://mysql.rjweb.org/doc.php/mysql_analysis#slow_queries_and_slowlog There are multiple signs of queries that may need improving -- perhaps better indexing, perhaps better formulation. The slowlog will help us locate which ones to focus on.

Is this server part of a Galera cluster? I may have further questions or comments depending on the answer.

Max_used_connections reached 199. This is rather high. Hopefully tackling the slow queries will bring this down.

Details and other observations:

( (key_buffer_size / 0.20 + innodb_buffer_pool_size / 0.70) / _ram ) = (32M / 0.20 + 12288M / 0.70) / 65536M = 27.0% -- Most of available ram should be made available for caching. -- http://mysql.rjweb.org/doc.php/memory

( table_open_cache ) = 4,000 -- Number of table descriptors to cache -- 700 might be good in your case. (Open_tables = 535)

( innodb_lru_scan_depth ) = 1,024 -- "InnoDB: page_cleaner: 1000ms intended loop took ..." may be fixed by lowering lru_scan_depth

( Innodb_os_log_written ) = 8,213,856,256 / 37691 = 217926 /sec -- This is an indicator of how busy InnoDB is. -- Very busy InnoDB.

( Innodb_rows_deleted / Innodb_rows_inserted ) = 1,085,550 / 1658210 = 0.655 -- Churn -- "Don't queue it, just do it." (If MySQL is being used as a queue.)

( Innodb_row_lock_waits ) = 9,462 / 37691 = 0.25 /sec -- How often there is a delay in getting a row lock. -- May be caused by complex queries that could be optimized.

( innodb_flush_neighbors ) = 1 -- A minor optimization when writing blocks to disk. -- Use 0 for SSD drives; 1 for HDD.

( sync_binlog ) = 0 -- Use 1 for added security, at some cost of I/O =1 may lead to lots of "query end"; =0 may lead to "binlog at impossible position" and lose transactions in a crash, but is faster.

( innodb_thread_concurrency ) = 0 -- 0 = Let InnoDB decide the best for concurrency_tickets. -- Set to 0 or 64. This may cut back on CPU.

( innodb_print_all_deadlocks ) = innodb_print_all_deadlocks = OFF -- Whether to log all Deadlocks. -- If you are plagued with Deadlocks, turn this on. Caution: If you have lots of deadlocks, this may write a lot to disk.

( local_infile ) = local_infile = ON -- local_infile = ON is a potential security issue

( Created_tmp_disk_tables ) = 100,172 / 37691 = 2.7 /sec -- Frequency of creating disk "temp" tables as part of complex SELECTs -- increase tmp_table_size and max_heap_table_size. Check the rules for temp tables on when MEMORY is used instead of MyISAM. Perhaps minor schema or query changes can avoid MyISAM. Better indexes and reformulation of queries are more likely to help.

( Select_full_join ) = 228,142 / 37691 = 6.1 /sec -- joins without index -- Add suitable index(es) to tables used in JOINs.

( Select_scan ) = 451,720 / 37691 = 12 /sec -- full table scans -- Add indexes / optimize queries (unless they are tiny tables)

( Select_scan / Com_select ) = 451,720 / 4944883 = 9.1% -- % of selects doing full table scan. (May be fooled by Stored Routines.) -- Add indexes / optimize queries

( Sort_merge_passes ) = 7,711 / 37691 = 0.2 /sec -- Heafty sorts -- Increase sort_buffer_size and/or optimize complex queries.

( Com_insert + Com_delete + Com_delete_multi + Com_replace + Com_update + Com_update_multi ) = (2323860 + 36898 + 0 + 0 + 2307510 + 3) / 37691 = 123 /sec -- writes/sec -- 50 writes/sec + log flushes will probably max out I/O write capacity of normal drives

( binlog_format ) = binlog_format = STATEMENT -- STATEMENT/ROW/MIXED. ROW is preferred; it may become the default.

( wsrep_on ) = wsrep_on = OFF -- Galera: Turn on Replication -- Why is it OFF?

( innodb_autoinc_lock_mode ) = 1 -- Galera: desires 2 -- 2 = "interleaved"; 1 = "consecutive" is typical; 0 = "traditional".

( wsrep_log_conflicts ) = wsrep_log_conflicts = OFF -- If you get deadlock conflicts during COMMIT, this flag can be helpful.

( slow_query_log ) = slow_query_log = OFF -- Whether to log slow queries. (5.1.12)

( long_query_time ) = 10 -- Cutoff (Seconds) for defining a "slow" query. -- Suggest 2

( Connections ) = 619,511 / 37691 = 16 /sec -- Connections -- Increase wait_timeout; use pooling?

( Threads_created / Connections ) = 17,071 / 619511 = 2.8% -- Rapidity of process creation ( thread_cache_size / max_connections ) = 8 / 512 = 1.6% -- Increase thread_cache_size (non-Windows)

Abnormally small:

Acl_database_grants = 3 Acl_users = 5 Aria_pagecache_blocks_unused = 13,617 Innodb_background_log_sync = 36,995 Innodb_log_writes / Innodb_log_write_requests = 0.20% Innodb_master_thread_idle_loops = 512 Memory_used = 179.5MB

Abnormally large:

Aria_pagecache_write_requests = 682 /sec Aria_pagecache_writes = 270 /sec Aria_transaction_log_syncs = 1,238 Com_show_binlogs = 12 /HR Com_show_engine_status = 83 /HR Com_show_plugins = 83 /HR Feature_locale = 3.1 /HR Handler_delete = 74 /sec Handler_discover = 5 /HR Handler_icp_attempts = 60233 /sec Handler_icp_match = 59945 /sec Handler_tmp_update = 5583 /sec Handler_tmp_write = 11043 /sec Innodb_buffer_pool_bytes_data = 296959 /sec Innodb_buffer_pool_pages_dirty = 61,297 Innodb_buffer_pool_write_requests = 5262 /sec Innodb_ibuf_merged_deletes = 0.32 /sec Innodb_rows_updated = 358 /sec Rows_tmp_read = 7844 /sec Select_range = 59 /sec Select_range / Com_select = 44.7% Sort_rows = 9375 /sec Tc_log_page_size = 4,096 Update_scan = 123,123 innodb_read_io_threads = 64 max_relay_log_size = 100MB

Abnormal strings:

innodb_default_row_format = compact innodb_fast_shutdown = 1 myisam_stats_method = NULLS_UNEQUAL

Changes made by OP

innodb_flush_neighbors 1 | innodb_flush_neighbors 0 long_query_time 10.000000 | long_query_time 1.000000 open_files_limit 8551 | open_files_limit 2565 pseudo_thread_id 16370 | pseudo_thread_id 32010 rand_seed1 357956615 | rand_seed1 386239716 rand_seed2 975103936 | rand_seed2 465882405 slow_query_log OFF | slow_query_log ON slow_query_log_file prod4-slo | slow_query_log_file /var/ table_open_cache 4000 | table_open_cache 700 thread_cache_size 8 | thread_cache_size 50 timestamp 1561288673.757972 | timestamp 1561333380.04
Improve this answer
Sign up to request clarification or add additional context in comments.

1 Comment

Thank you! I have enabled slowlog and changed some of the settings, and made it so that some slow queries are more infrequent, new settings are here: pastebin.com/N55AzWFw also, this server is not part of a galera cluster
1

After Rick James suggesting that some default settings are weird I decided to upgrade to official (1:10.2.25+maria~cosmic) instead of ubuntu distributed packages and the memory problem seems to have gone away without any changes to config.

 PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 2736 mysql 20 0 27.4g 15.0g 22624 S 141.7 23.9 384:24.58 /usr/sbin/mysqld

current variables: https://pastebin.com/7C96E6J4

current status: https://pastebin.com/G55ydpJy

Additional data as requested by Wilson Hauck:

we are using 2x Samsung SSD 850 EVO 500GB in raid 1 on on an i7-6700K

iostat -xm 5 3: Linux 4.18.0-20-generic (prod4) 07/04/2019 _x86_64_ (8 CPU) avg-cpu: %user %nice %system %iowait %steal %idle 7.10 0.08 1.43 1.53 0.00 89.86 Device r/s w/s rMB/s wMB/s rrqm/s wrqm/s %rrqm %wrqm r_await w_await aqu-sz rareq-sz wareq-sz svctm %util sdb 6.35 205.97 0.41 4.21 1.79 63.35 21.94 23.52 0.33 1.85 0.67 66.31 20.92 1.19 25.22 sda 14.63 205.95 0.70 4.17 2.58 62.76 14.98 23.36 0.19 1.43 0.58 48.67 20.73 1.06 23.46 md0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 3.47 489.75 0.00 0.00 md2 19.86 232.03 0.84 4.23 0.00 0.00 0.00 0.00 0.00 0.00 0.00 43.43 18.65 0.00 0.00 md1 1.33 2.51 0.01 0.01 0.00 0.00 0.00 0.00 0.00 0.00 0.00 4.00 4.00 0.00 0.00 avg-cpu: %user %nice %system %iowait %steal %idle 9.38 0.00 1.61 2.18 0.00 86.83 Device r/s w/s rMB/s wMB/s rrqm/s wrqm/s %rrqm %wrqm r_await w_await aqu-sz rareq-sz wareq-sz svctm %util sdb 0.00 352.80 0.00 6.88 0.00 57.60 0.00 14.04 0.00 0.26 0.40 0.00 19.97 0.88 31.12 sda 0.00 353.00 0.00 6.88 0.00 57.40 0.00 13.99 0.00 0.26 0.39 0.00 19.96 0.86 30.48 md0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 md2 0.00 370.80 0.00 6.87 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 18.98 0.00 0.00 md1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 avg-cpu: %user %nice %system %iowait %steal %idle 8.00 0.00 1.69 2.16 0.00 88.14 Device r/s w/s rMB/s wMB/s rrqm/s wrqm/s %rrqm %wrqm r_await w_await aqu-sz rareq-sz wareq-sz svctm %util sdb 0.00 387.60 0.00 7.49 0.00 76.20 0.00 16.43 0.00 0.24 0.42 0.00 19.78 0.83 32.16 sda 0.00 387.60 0.00 7.49 0.00 76.20 0.00 16.43 0.00 0.31 0.42 0.00 19.78 0.82 31.60 md0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 md2 0.00 422.20 0.00 7.48 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 18.15 0.00 0.00 md1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
Improve this answer

1 Comment

v 10.2.25 has worked in your favor. Thanks for current variables and status. Do you have SSD or rotating HDD? Also could you post TEXT results from LX Command Prompt of iostat -xm 5 3 for IOPS by device and core/cpu count?
1

Rate Per Second=RPS - Suggestions to consider for your my.cnf [mysqld] section

thread_cache_size=256 # from 50 per 10.n.nn MaridDB refman minimum innodb_io_capacity=1900 # from 400 to enable higher SSD IOPS innodb_lru_scan_depth=100 # from 1024 to conserve 90% of CPU cycles used for function innodb_buffer_pool_size=24G # from 12G to reduce innodb_buffer_pool_reads RPS of 14 innodb_flushing_avg_loops=5 # from 30 to reduce innodb_buffer_pool_pages_dirty of 61,297

Disclaimer: I am the content author of website mentioned on my profile, Network profile where we have free Utility Scripts to improve performance and additional suggestions.

Improve this answer

Comments

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.