Local node master listener

* Fixed an issue where dynamic update to minimum_master_nodes settings would not take immediate effect * Added LocalNodeMasterListener support to the ClusterService. Enables listening to when the local node becomes/stopped being a master

Author: uboness

src/main/java/org/elasticsearch/cluster/ClusterService.java

@@ -72,6 +72,16 @@ public interface ClusterService extends LifecycleComponent<ClusterService> {
  */
  void remove(ClusterStateListener listener);
 
+ /**
+ * Add a listener for on/off local node master events
+ */
+ void add(LocalNodeMasterListener listener);
+
+ /**
+ * Remove the given listener for on/off local master events
+ */
+ void remove(LocalNodeMasterListener listener);
+
  /**
  * Adds a cluster state listener that will timeout after the provided timeout.
  */

src/main/java/org/elasticsearch/cluster/LocalNodeMasterListener.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster;
+
+/**
+ * Enables listening to master changes events of the local node (when the local node becomes the master, and when the local
+ * node cease being a master).
+ */
+public interface LocalNodeMasterListener {
+
+ /**
+ * Called when local node is elected to be the master
+ */
+ void onMaster();
+
+ /**
+ * Called when the local node used to be the master, a new master was elected and it's no longer the local node.
+ */
+ void offMaster();
+
+ /**
+ * The name of the executor that the implementation of the callbacks of this lister should be executed on. The thread
+ * that is responsible for managing instances of this lister is the same thread handling the cluster state events. If
+ * the work done is the callbacks above is inexpensive, this value may be {@link org.elasticsearch.threadpool.ThreadPool.Names#SAME SAME}
+ * (indicating that the callbaks will run on the same thread as the cluster state events are fired with). On the other hand,
+ * if the logic in the callbacks are heavier and take longer to process (or perhaps involve blocking due to IO operations),
+ * prefer to execute them on a separte more appropriate executor (eg. {@link org.elasticsearch.threadpool.ThreadPool.Names#GENERIC GENERIC}
+ * or {@link org.elasticsearch.threadpool.ThreadPool.Names#MANAGEMENT MANAGEMENT}).
+ *
+ * @return The name of the executor that will run the callbacks of this listener.
+ */
+ String executorName();
+
+}
+

src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java

@@ -43,10 +43,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Queue;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
 
 import static java.util.concurrent.Executors.newSingleThreadExecutor;
 import static org.elasticsearch.cluster.ClusterState.Builder;
@@ -75,6 +72,7 @@ public class InternalClusterService extends AbstractLifecycleComponent<ClusterSe
  private final List<ClusterStateListener> priorityClusterStateListeners = new CopyOnWriteArrayList<ClusterStateListener>();
  private final List<ClusterStateListener> clusterStateListeners = new CopyOnWriteArrayList<ClusterStateListener>();
  private final List<ClusterStateListener> lastClusterStateListeners = new CopyOnWriteArrayList<ClusterStateListener>();
+ private final LocalNodeMasterListeners localNodeMasterListeners;
 
  private final Queue<NotifyTimeout> onGoingTimeouts = ConcurrentCollections.newQueue();
 
@@ -97,6 +95,8 @@ public InternalClusterService(Settings settings, DiscoveryService discoveryServi
  this.nodeSettingsService.setClusterService(this);
 
  this.reconnectInterval = componentSettings.getAsTime("reconnect_interval", TimeValue.timeValueSeconds(10));
+
+ localNodeMasterListeners = new LocalNodeMasterListeners(threadPool);
  }
 
  public NodeSettingsService settingsService() {
@@ -112,6 +112,7 @@ public void addInitialStateBlock(ClusterBlock block) throws ElasticSearchIllegal
 
  @Override
  protected void doStart() throws ElasticSearchException {
+ add(localNodeMasterListeners);
  this.clusterState = newClusterStateBuilder().blocks(initialBlocks).build();
  this.updateTasksExecutor = newSingleThreadExecutor(daemonThreadFactory(settings, "clusterService#updateTask"));
  this.reconnectToNodes = threadPool.schedule(reconnectInterval, ThreadPool.Names.GENERIC, new ReconnectToNodes());
@@ -130,6 +131,7 @@ protected void doStop() throws ElasticSearchException {
  } catch (InterruptedException e) {
  // ignore
  }
+ remove(localNodeMasterListeners);
  }
 
  @Override
@@ -175,6 +177,16 @@ public void remove(ClusterStateListener listener) {
  }
  }
 
+ @Override
+ public void add(LocalNodeMasterListener listener) {
+ localNodeMasterListeners.add(listener);
+ }
+
+ @Override
+ public void remove(LocalNodeMasterListener listener) {
+ localNodeMasterListeners.remove(listener);
+ }
+
  public void add(TimeValue timeout, final TimeoutClusterStateListener listener) {
  if (lifecycle.stoppedOrClosed()) {
  listener.onClose();
@@ -398,4 +410,75 @@ public void run() {
  private boolean nodeRequiresConnection(DiscoveryNode node) {
  return localNode().shouldConnectTo(node);
  }
+
+ private static class LocalNodeMasterListeners implements ClusterStateListener {
+
+ private final List<LocalNodeMasterListener> listeners = new CopyOnWriteArrayList<LocalNodeMasterListener>();
+ private final ThreadPool threadPool;
+ private volatile boolean master = false;
+
+ private LocalNodeMasterListeners(ThreadPool threadPool) {
+ this.threadPool = threadPool;
+ }
+
+ @Override
+ public void clusterChanged(ClusterChangedEvent event) {
+ if (!master && event.localNodeMaster()) {
+ master = true;
+ for (LocalNodeMasterListener listener : listeners) {
+ Executor executor = threadPool.executor(listener.executorName());
+ executor.execute(new OnMasterRunnable(listener));
+ }
+ return;
+ }
+
+ if (master && !event.localNodeMaster()) {
+ master = false;
+ for (LocalNodeMasterListener listener : listeners) {
+ Executor executor = threadPool.executor(listener.executorName());
+ executor.execute(new OffMasterRunnable(listener));
+ }
+ }
+ }
+
+ private void add(LocalNodeMasterListener listener) {
+ listeners.add(listener);
+ }
+
+ private void remove(LocalNodeMasterListener listener) {
+ listeners.remove(listener);
+ }
+
+ private void clear() {
+ listeners.clear();
+ }
+ }
+
+ private static class OnMasterRunnable implements Runnable {
+
+ private final LocalNodeMasterListener listener;
+
+ private OnMasterRunnable(LocalNodeMasterListener listener) {
+ this.listener = listener;
+ }
+
+ @Override
+ public void run() {
+ listener.onMaster();
+ }
+ }
+
+ private static class OffMasterRunnable implements Runnable {
+
+ private final LocalNodeMasterListener listener;
+
+ private OffMasterRunnable(LocalNodeMasterListener listener) {
+ this.listener = listener;
+ }
+
+ @Override
+ public void run() {
+ listener.offMaster();
+ }
+ }
 }

src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java

@@ -142,7 +142,8 @@ public ZenDiscovery(Settings settings, ClusterName clusterName, ThreadPool threa
 
  logger.debug("using ping.timeout [{}], master_election.filter_client [{}], master_election.filter_data [{}]", pingTimeout, masterElectionFilterClientNodes, masterElectionFilterDataNodes);
 
- this.electMaster = new ElectMasterService(settings, nodeSettingsService);
+ this.electMaster = new ElectMasterService(settings);
+ nodeSettingsService.addListener(new ApplySettings());
 
  this.masterFD = new MasterFaultDetection(settings, threadPool, transportService, this);
  this.masterFD.addListener(new MasterNodeFailureListener());
@@ -410,6 +411,34 @@ public void clusterStateProcessed(ClusterState clusterState) {
  });
  }
 
+ private void handleMinimumMasterNodesChanged(final int minimumMasterNodes) {
+ if (lifecycleState() != Lifecycle.State.STARTED) {
+ // not started, ignore a node failure
+ return;
+ }
+ if (!master) {
+ // nothing to do here...
+ return;
+ }
+ clusterService.submitStateUpdateTask("zen-disco-minimum_master_nodes_changed", new ProcessedClusterStateUpdateTask() {
+ @Override
+ public ClusterState execute(ClusterState currentState) {
+ final int prevMinimumMasterNode = ZenDiscovery.this.electMaster.minimumMasterNodes();
+ ZenDiscovery.this.electMaster.minimumMasterNodes(minimumMasterNodes);
+ // check if we have enough master nodes, if not, we need to move into joining the cluster again
+ if (!electMaster.hasEnoughMasterNodes(currentState.nodes())) {
+ return rejoin(currentState, "not enough master nodes on change of minimum_master_nodes from [" + prevMinimumMasterNode + "] to [" + minimumMasterNodes + "]");
+ }
+ return currentState;
+ }
+
+ @Override
+ public void clusterStateProcessed(ClusterState clusterState) {
+ sendInitialStateEventIfNeeded();
+ }
+ });
+ }
+
  private void handleMasterGone(final DiscoveryNode masterNode, final String reason) {
  if (lifecycleState() != Lifecycle.State.STARTED) {
  // not started, ignore a master failure
@@ -754,12 +783,12 @@ static class RejoinClusterRequest implements Streamable {
 
  @Override
  public void readFrom(StreamInput in) throws IOException {
- fromNodeId = in.readOptionalUTF();
+ fromNodeId = in.readOptionalString();
  }
 
  @Override
  public void writeTo(StreamOutput out) throws IOException {
- out.writeOptionalUTF(fromNodeId);
+ out.writeOptionalString(fromNodeId);
  }
  }
 
@@ -793,4 +822,15 @@ public String executor() {
  }
  }
 
+ class ApplySettings implements NodeSettingsService.Listener {
+ @Override
+ public void onRefreshSettings(Settings settings) {
+ int minimumMasterNodes = settings.getAsInt("discovery.zen.minimum_master_nodes", ZenDiscovery.this.electMaster.minimumMasterNodes());
+ if (minimumMasterNodes != ZenDiscovery.this.electMaster.minimumMasterNodes()) {
+ logger.info("updating discovery.zen.minimum_master_nodes from [{}] to [{}]", ZenDiscovery.this.electMaster.minimumMasterNodes(), minimumMasterNodes);
+ handleMinimumMasterNodesChanged(minimumMasterNodes);
+ }
+ }
+ }
+
 }

src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java

@@ -24,7 +24,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.node.settings.NodeSettingsService;
 
 import java.util.Collections;
 import java.util.Comparator;
@@ -44,11 +43,18 @@ public class ElectMasterService extends AbstractComponent {
 
  private volatile int minimumMasterNodes;
 
- public ElectMasterService(Settings settings, NodeSettingsService nodeSettingsService) {
+ public ElectMasterService(Settings settings) {
  super(settings);
  this.minimumMasterNodes = settings.getAsInt("discovery.zen.minimum_master_nodes", -1);
  logger.debug("using minimum_master_nodes [{}]", minimumMasterNodes);
- nodeSettingsService.addListener(new ApplySettings());
+ }
+
+ public void minimumMasterNodes(int minimumMasterNodes) {
+ this.minimumMasterNodes = minimumMasterNodes;
+ }
+
+ public int minimumMasterNodes() {
+ return minimumMasterNodes;
  }
 
  public boolean hasEnoughMasterNodes(Iterable<DiscoveryNode> nodes) {
@@ -111,17 +117,6 @@ private List<DiscoveryNode> sortedMasterNodes(Iterable<DiscoveryNode> nodes) {
  return possibleNodes;
  }
 
- class ApplySettings implements NodeSettingsService.Listener {
- @Override
- public void onRefreshSettings(Settings settings) {
- int minimumMasterNodes = settings.getAsInt("discovery.zen.minimum_master_nodes", ElectMasterService.this.minimumMasterNodes);
- if (minimumMasterNodes != ElectMasterService.this.minimumMasterNodes) {
- logger.info("updating [discovery.zen.minimum_master_nodes] from [{}] to [{}]", ElectMasterService.this.minimumMasterNodes, minimumMasterNodes);
- ElectMasterService.this.minimumMasterNodes = minimumMasterNodes;
- }
- }
- }
-
  private static class NodeComparator implements Comparator<DiscoveryNode> {
 
  @Override