order
This paper focuses on the broker monitor of carrera
BrokerMonitor
DDMQ/carrera-monitor/src/main/java/com/xiaojukeji/carrera/monitor/broker/BrokerMonitor.java
public class BrokerMonitor extends BaseConfigMonitor { private final static Logger LOGGER = LoggerFactory.getLogger(BrokerMonitor.class); private ExecutorService executor = ExecutorUtils.newFixedThreadPool(100, "BrokerMonitor", 200); private BrokerMonitorItem monitorItem = null; @Override protected void initMonitor(String broker, BrokerConfig brokerConfig) throws Exception { doMonitor(broker, brokerConfig); } public BrokerMonitor(MonitorConfig monitorConfig) { super("Broker", monitorConfig); } private void doMonitor(String broker, BrokerConfig config) throws InterruptedException { if (monitorItem != null) { // stop first. LOGGER.info("Stop old monitor broker: {}", broker); monitorItem.stop(); } BrokerMonitorItem item = new BrokerMonitorItem(broker, config); try { item.start(); } catch (Exception e) { LOGGER.error("broker monitor start exception, broker=" + broker, e); } } @Override public void shutdown() { ExecutorUtils.shutdown(executor); monitorItem.stop(); super.shutdown(); } //...... }
- The BrokerMonitor inherits the BaseConfigMonitor. Its initMonitor method executes doMonitor, and its shutdown will shut down the executor and execute monitorItem.stop(). If the doMonitor method judges that monitorItem is not null, first execute monitorItem.stop(), then create the BrokerMonitorItem and execute its start method
BrokerMonitorItem
DDMQ/carrera-monitor/src/main/java/com/xiaojukeji/carrera/monitor/broker/BrokerMonitor.java
class BrokerMonitorItem { private String broker; private BrokerConfig config; private volatile boolean isRunning = false; private ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(); public BrokerMonitorItem(String broker, BrokerConfig config) { this.broker = broker; this.config = config; } public void start() { isRunning = true; scheduledExecutor.submit(() -> { while (isRunning) { monitorNamesvr(); monitorBroker(); try { Thread.sleep(10000); } catch (InterruptedException e) { e.printStackTrace(); } LOGGER.info("broker<{}> [Active]", broker); } }); } public void stop() { isRunning = false; ExecutorUtils.shutdown(scheduledExecutor); } private void monitorBroker() { if (MapUtils.isEmpty(config.getBrokers()) || StringUtils.isBlank(config.getBrokerClusterAddrs())) { return; } String nameSvr = config.getBrokerClusterAddrs().split(";")[0]; // use first namesvr. for (Map.Entry<String, Set<String>> entry : config.getBrokers().entrySet()) { String master = entry.getKey(); Set<String> slaves = entry.getValue(); executor.execute(() -> { int j = 0; for (; j < 2; ++j) { try { long masterOffset = Utils.checkReceive(broker, nameSvr, master); if (masterOffset <= 0) { continue; } Utils.checkSend(broker, nameSvr, master); if (CollectionUtils.isNotEmpty(slaves)) { for (String slave : slaves) { long slaveOffset = Utils.checkReceive(broker, nameSvr, slave); LOGGER.info("ReplicaDelayCheck broker={}, address={}->{}, masterOffset={}, slaveOffset={}, delayNum={}", broker, master.split(":")[0], slave.split(":")[0], masterOffset, slaveOffset, (masterOffset - slaveOffset)); if (slaveOffset <= 0) { continue; } if (masterOffset - slaveOffset > 60) { LOGGER.error(String.format("[AlarmReplicaDelayErr] broker=%s, address=%s->%s, delayNum=%s", broker, master.split(":")[0], slave.split(":")[0], (masterOffset - slaveOffset))); } } } break; } catch (Throwable e) { LOGGER.error("broker check broker exception, broker=" + broker, e); } try { Thread.sleep(1000); } catch (InterruptedException e) { } } if (j == 2) { LOGGER.error(String.format("[AlarmCheckBrokerErr] broker=%s, namesvr=%s", broker, nameSvr)); } }); } } private void monitorNamesvr() { if (StringUtils.isBlank(config.getBrokerClusterAddrs())) { LOGGER.info("broker:{}, brokerClusterAddrs is empty", config.getBrokerCluster()); return; } for (String nameSvr : config.getBrokerClusterAddrs().split(";")) { executor.execute(() -> { int j = 0; for (; j < 2; ++j) { try { Utils.checkNameSvr(nameSvr, broker); LOGGER.info(String.format("[NameSvrCheck] broker=%s, namesvr=%s", broker, nameSvr)); break; } catch (Throwable e) { LOGGER.error("broker checkNameSvr exception, broker=" + broker + ", namesvr=" + nameSvr, e); } try { Thread.sleep(1000); } catch (InterruptedException e) { LOGGER.error("broker checkNameSvr Thread.sleep exception, broker=" + broker, e); } } if (j == 2) { LOGGER.error(String.format("[AlarmNameSvrErr] broker=%s, namesvr=%s", broker, nameSvr)); } }); } } }
- Broker monitoritem's start method will execute a runnable asynchronously, which will continuously execute monitorNamesvr and monitorBroker methods; monitorNamesvr method will traverse nameSvr to execute Utils.checkNameSvr(nameSvr, broker); monitorBroker method will traverse config.getBrokers().entrySet(), and execute Utils.checkReceive(broker, nameSvr, master) and utils.checksend (broker Ker, nameSvr, Master). For slaves, execute Utils.checkReceive(broker, nameSvr, slave)
Utils
DDMQ/carrera-monitor/src/main/java/com/xiaojukeji/carrera/monitor/broker/Utils.java
public class Utils { private static final Logger logger = LoggerFactory.getLogger(Utils.class); private static final Map<String, DefaultMQAdminExt> mqAdminExtMap = new ConcurrentHashMap<>(); private static final Map<String, DefaultMQPullConsumer> nameSvrCheckMap = new ConcurrentHashMap<>(); private static final Map<String, DefaultMQPullConsumer> brokerReceiveCheckMap = new ConcurrentHashMap<>(); private static final Map<String, DefaultMQProducer> brokerSendCheckMap = new ConcurrentHashMap<>(); //...... public static void checkNameSvr(String nameSvr, String cluster) throws MQClientException, InterruptedException { getNameSvrCheckConsumer(nameSvr, cluster).getDefaultMQPullConsumerImpl().fetchPublishMessageQueues("SELF_TEST_TOPIC"); } public static long checkReceive(String cluster, String nameSvr, String address) throws MQClientException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException, InterruptedException, RemotingException, MQBrokerException { DefaultMQPullConsumer consumer = getReceiveCheckConsumer(nameSvr, cluster, address); Field f1 = DefaultMQPullConsumerImpl.class.getDeclaredField("mQClientFactory"); f1.setAccessible(true); MQClientInstance instance = (MQClientInstance) f1.get(consumer.getDefaultMQPullConsumerImpl()); Field f = MQClientInstance.class.getDeclaredField("brokerAddrTable"); f.setAccessible(true); Field f2 = MQClientInstance.class.getDeclaredField("scheduledExecutorService"); f2.setAccessible(true); ScheduledExecutorService service = (ScheduledExecutorService) f2.get(instance); service.shutdown(); service.awaitTermination(1000, TimeUnit.SECONDS); ConcurrentHashMap<String, HashMap<Long, String>> map = (ConcurrentHashMap<String, HashMap<Long, String>>) f.get(instance); HashMap<Long, String> addresses = new HashMap<>(); addresses.put(0L, address); map.put("rmqmonitor_" + address, addresses); MessageQueue queue = new MessageQueue("SELF_TEST_TOPIC", "rmqmonitor_" + address, 0); boolean pullOk = false; long maxOffset = -1; for (int i = 0; i < 2; ++i) { try { maxOffset = consumer.getDefaultMQPullConsumerImpl().maxOffset(queue); PullResult result = consumer.pull(queue, "*", maxOffset > 100 ? maxOffset - 10 : 0, 1); if (result.getPullStatus() == PullStatus.FOUND) { pullOk = true; break; } else if(result.getPullStatus() == PullStatus.NO_NEW_MSG) { checkSend(cluster, nameSvr, address); continue; } logger.warn("pull result failed, PullResult={}, cluster={}, namesvr={}, address={}", result, cluster, nameSvr, address); } catch (Throwable e) { logger.error("pull exception, cluster={}, namesvr={}, address={}", cluster, nameSvr, address, e); } Thread.sleep(1000); } if (!pullOk) { logger.error(String.format("[AlarmPullErr] cluster=%s, broker=%s", cluster, address)); } else { logger.info("AlarmPullCheck cluster={}, broker={}", cluster, address); } return maxOffset; } public static void checkSend(String cluster, String nameSvr, String address) throws MQClientException, NoSuchFieldException, SecurityException, InterruptedException, IllegalArgumentException, IllegalAccessException, UnsupportedEncodingException, MQBrokerException, RemotingException { if (!isBrokerTopicWritable(cluster, nameSvr, address)) { return; } DefaultMQProducer producer = getSendCheckProducer(nameSvr, cluster, address); MQClientInstance instance = producer.getDefaultMQProducerImpl().getmQClientFactory(); Field f = MQClientInstance.class.getDeclaredField("brokerAddrTable"); f.setAccessible(true); Field f2 = MQClientInstance.class.getDeclaredField("scheduledExecutorService"); f2.setAccessible(true); ScheduledExecutorService service = (ScheduledExecutorService) f2.get(instance); service.shutdown(); service.awaitTermination(1000, TimeUnit.SECONDS); ConcurrentHashMap<String, HashMap<Long, String>> map = (ConcurrentHashMap<String, HashMap<Long, String>>) f .get(instance); HashMap<Long, String> addresses = new HashMap<>(); addresses.put(0L, address); map.put("rmqmonitor_" + address, addresses); MessageQueue queue = new MessageQueue("SELF_TEST_TOPIC", "rmqmonitor_" + address, 0); boolean sendOk = false; SendResult sendResult = null; for (int i = 0; i < 2; i++) { try { Message msg = new Message("SELF_TEST_TOPIC", // topic "TagA", // tag ("Hello RocketMQ " + i).getBytes()// body ); sendResult = producer.send(msg, queue); if (sendResult.getSendStatus() == SendStatus.SEND_OK || sendResult.getSendStatus() == SLAVE_NOT_AVAILABLE) { sendOk = true; break; } logger.warn("send result failed, SendResult={}, cluster={}, namesvr={}, address={}", sendResult, cluster, nameSvr, address); } catch (Exception e) { logger.error("send exception, cluster={}, namesvr={}, address={}", cluster, nameSvr, address, e); } Thread.sleep(1000); } // Call the police if (!sendOk) { logger.error(String.format("[AlarmSendErr] cluster=%s, broker=%s, result=%s", cluster, address, sendResult == null ? "null" : sendResult.toString())); } else { logger.info("AlarmSendCheck cluster={}, broker={}, result={}", cluster, address, sendResult.toString()); } } //...... }
- checkNameSvr method executes getNameSvrCheckConsumer(nameSvr,cluster).getDefaultMQPullConsumerImpl().fetchPublishMessageQueues method; checkReceive method mainly pulls data from self test topic; checkSend method mainly sends data to self test topic
Summary
The BrokerMonitor inherits the BaseConfigMonitor. Its initMonitor method executes doMonitor, and its shutdown will shut down the executor and execute monitorItem.stop(). If the doMonitor method judges that monitorItem is not null, first execute monitorItem.stop(), then create the BrokerMonitorItem and execute its start method