Asynchronous call to mqtt message processing through annotation based on SpringBoot
Use background
In the production environment, as mqtt producers produce more and more messages, it may lead to message accumulation. Therefore, consumers need to consume quickly
One of the schemes is to use asynchronous threads to speed up the consumption of messages
We can modify the original mqtt tools
First, create a class MqttMessageListener and inherit IMqttMessageListener to implement messageArrived, which is used to process these messages (business writing)
Then rewrite the mqtt client subscription method, inject MqttMessageListener, and add the parameter in the subscription method
Then start the asynchronous thread in the startup class, write a configuration class to configure the thread pool parameters, and add @ Async to messageArrived to start the asynchronous thread call
code implementation
Basic code
Refers to the code that does not open the thread pool
- MqttPushClient mainly defines connection parameters
import org.eclipse.paho.client.mqttv3.IMqttToken;
import org.eclipse.paho.client.mqttv3.MqttClient;
import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* @Author
* @Date
* @Description Connect to EMQ X server, get mqtt connection and publish message
*/
@Component
public class MqttPushClient{
private static final Logger log = LoggerFactory.getLogger(MqttPushClient.class);
@Autowired
private PushCallback pushCallback;
private static MqttClient client;
public static void setClient(MqttClient client) {
MqttPushClient.client = client;
}
public static MqttClient getClient() {
return client;
}
public void connect(String host, String clientID, String username, String password, int timeout, int keepalive, List<String> topicList) {
MqttClient client;
try {
client = new MqttClient(host, clientID, new MemoryPersistence());
MqttConnectOptions options = new MqttConnectOptions();
options.setCleanSession(true);
if (username != null) {
options.setUserName(username);
}
if (password != null) {
options.setPassword(password.toCharArray());
}
options.setConnectionTimeout(timeout);
options.setKeepAliveInterval(keepalive);
MqttPushClient.setClient(client);
try {
//Set callback class
client.setCallback(pushCallback);
//client.connect(options);
IMqttToken iMqttToken = client.connectWithResult(options);
boolean complete = iMqttToken.isComplete();
log.info("MQTT connect"+(complete?"success":"fail"));
/** Subscribe to topics**/
for (String topic : topicList) {
log.info("Connection subscription topic:{}", topic);
client.subscribe(topic, 0);
}
} catch (Exception e) {
e.printStackTrace();
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
- PushCallback callback class, which realizes reconnection, message sending monitoring and message receiving monitoring
import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;
import org.eclipse.paho.client.mqttv3.MqttCallback;
import org.eclipse.paho.client.mqttv3.MqttMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
/**
* @Author
* @Date
* @Description Message callback to process the received message
*/
@Component
public class PushCallback implements MqttCallback {
private static final Logger log = LoggerFactory.getLogger(PushCallback.class);
@Autowired
private MqttConfiguration mqttConfiguration;
@Autowired
private MqttTopic mqttTopic;
@Override
public void connectionLost(Throwable cause) { // After the connection is lost, it is usually reconnected here
log.info("Disconnected, reconnecting");
MqttPushClient mqttPushClient = mqttConfiguration.getMqttPushClient();
if (null != mqttPushClient) {
mqttPushClient.connect(mqttConfiguration.getHost(), mqttConfiguration.getClientid(), mqttConfiguration.getUsername(),
mqttConfiguration.getPassword(), mqttConfiguration.getTimeout(), mqttConfiguration.getKeepalive(), mqttConfiguration.getTopic());
log.info("Reconnected");
}
}
/**
* Send a message, and the processing method after the message arrives
* @param token
*/
@Override
public void deliveryComplete(IMqttDeliveryToken token) {
int messageId = token.getMessageId();
String[] topics = token.getTopics();
log.info("Message sending completed,messageId={},topics={}",messageId,topics.toString());
}
/**
* Message processing method received by subscription topic
* @param topic
* @param message
*/
@Override
public void messageArrived(String topic, MqttMessage message) {
// The messages obtained after subscribing will be executed here, and there will be output on the console
String messageStr = new String(message.getPayload());
// messageDistribute.distribute(topic, messageStr);
log.info("Received subject:" + topic + ";Information received:" + messageStr);
}
}
- MqttConfiguration configures mqtt related parameters and initializes the connection (mqtt starts here)
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* @Author
* @Date mqtt Configuration and connection
* @Description
*/
@Slf4j
@Component
@Configuration
@ConfigurationProperties(MqttConfiguration.PREFIX)
public class MqttConfiguration {
@Autowired
private MqttPushClient mqttPushClient;
/**
* Specify the property name prefix in the configuration file application-local.properties
*/
public static final String PREFIX = "std.mqtt";
private String host;
private String clientId;
private String userName;
private String password;
private int timeout;
private int keepAlive;
private List<String> topic;
public String getClientid() {
return clientId;
}
public void setClientid(String clientid) {
this.clientId = clientid;
}
public String getUsername() {
return userName;
}
public void setUsername(String username) {
this.userName = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public int getTimeout() {
return timeout;
}
public void setTimeout(int timeout) {
this.timeout = timeout;
}
public int getKeepalive() {
return keepAlive;
}
public void setKeepalive(int keepalive) {
this.keepAlive = keepalive;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public List<String> getTopic() {
return topic;
}
public void setTopic(List<String> topic) {
this.topic = topic;
}
/**
* Connect to mqtt server to get mqtt connection
* @return
*/
@Bean
public MqttPushClient getMqttPushClient() {
//Connect to mqtt server to get mqtt connection
mqttPushClient.connect(host, clientId, userName, password, timeout, keepAlive, topic);
return mqttPushClient;
}
}
properties.yml configuration file
std.mqtt:
host: tcp://x.x.x.x:1883
username: your_username
password: your_password
#MQTT connection server default client ID
clientid: your_clientid
#connection timed out
timeout: 1000
# deviceId
deviceId: your_deviceId
# mqtt-topic
topic[0]: your_tpoic
- Topicooperation defines the method of publishing and subscribing
import org.eclipse.paho.client.mqttv3.MqttClient;
import org.eclipse.paho.client.mqttv3.MqttException;
import org.eclipse.paho.client.mqttv3.MqttMessage;
import org.eclipse.paho.client.mqttv3.MqttTopic;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
/**
* @Author chy
*/
public class TopicOperation {
private static final Logger log = LoggerFactory.getLogger(TopicOperation.class);
/**
* Subscribe to topics
* @param topic Subject name
*/
public static void subscribe(String topic) {
try {
MqttClient client = MqttPushClient.getClient();
if (client == null) {
return;
};
client.subscribe(topic, 0);
log.info("Subscribe to topics:{}",topic);
} catch (MqttException e) {
e.printStackTrace();
}
}
/**
* Publish theme
*
* @param topic
* @param pushMessage
*/
public static void publish(String topic, String pushMessage) {
log.info("SEND TO MQTT -- topic : {}, message : {}", topic, pushMessage);
MqttMessage message = new MqttMessage();
message.setQos(0);
// Non persistent
message.setRetained(false);
message.setPayload(pushMessage.getBytes());
MqttClient client = MqttPushClient.getClient();
if (client == null) {
return;
};
MqttTopic mTopic = client.getTopic(topic);
if (null == mTopic) {
log.error("Subject does not exist:{}",mTopic);
}
try {
mTopic.publish(message);
} catch (Exception e) {
log.error("mqtt Sending message exception:",e);
}
}
}
- Defines topics related to publications and subscriptions
import com.sxd.onlinereservation.exception.BusinessException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
/**
* @Author
* @Date topic name
* @Description
*/
@Component
public class MqttTopic {
@Value("${std.mqtt.deviceId}")
private String[] deviceId;
public String getSubscribeTopic(String type){
switch (type){
case "appointTopic":
return String.format("/v1/%s/service/appointTopic", deviceId[0]);
default:
throw new BusinessException("mqtt Subscription topic get error");
}
}
public String getPublishTopic(String type) {
switch (type){
//1.0 the interface immediately takes the number and publishes the topic
case "appointTopic":
return String.format("/v1/%s/service/appointTopic", deviceId[1]);
default:
throw new BusinessException("mqtt Publish topic get error");
}
}
}
ps: if you want to use this tool class to send and receive messages, see the demo below
//Message publishing operation
TopicOperation.publish(mqttTopic.getPublishTopic("appointTopic"), "Message body"));
//Message subscription operation
TopicOperation.subscribe(mqttTopic.getSubscribeTopic("appointTopic"), "Message body"));
Asynchronous thread processing implementation
summary
- Create a message listening class to listen to messages and perform business processing
- During the original subscription, inject and use the listening class created in step 1
- Open asynchronous threads through annotations and configure processing methods
Create a message listening class to listen to messages and perform business processing
@Slf4j
@Component
public class MqttMessageListener implements IMqttMessageListener {
@Resource
private BusinessService businessService;
@Autowired
private MqttTopic mqttTopic;
@Autowired
private ThreeCallmachineService threeCallmachineService;
@Autowired
private BusinessHallService businessHallService;
@Autowired
private BusinessMaterialService businessMaterialService;
@Autowired
private BusinessWaitService businessWaitService;
@Autowired
private AppointmentService appointmentService;
@Override
public void messageArrived(String topic, MqttMessage message) throws Exception {
String messageStr = new String(message.getPayload());
log.info("Received subject:" + topic + ";Information received:" + messageStr);
//Conduct business processing
}
}
During the original subscription, inject and use the listening class created in step 1
Inject MqttMessageListener, and add client.subscribe(topic, mqttMessageListener) during subscription;
- Modify MqttPushClient (required)
@Component
public class MqttPushClient{
private static final Logger log = LoggerFactory.getLogger(MqttPushClient.class);
@Autowired
private PushCallback pushCallback;
@Autowired //The injection operation is performed here
private MqttMessageListener mqttMessageListener;
private static MqttClient client;
public static void setClient(MqttClient client) {
MqttPushClient.client = client;
}
public static MqttClient getClient() {
return client;
}
public void connect(String host, String clientID, String username, String password, int timeout, int keepalive, List<String> topicList) {
MqttClient client;
try {
client = new MqttClient(host, clientID, new MemoryPersistence());
MqttConnectOptions options = new MqttConnectOptions();
options.setCleanSession(true);
if (username != null) {
options.setUserName(username);
}
if (password != null) {
options.setPassword(password.toCharArray());
}
options.setConnectionTimeout(timeout);
options.setKeepAliveInterval(keepalive);
MqttPushClient.setClient(client);
try {
//Set callback class
client.setCallback(pushCallback);
//client.connect(options);
IMqttToken iMqttToken = client.connectWithResult(options);
boolean complete = iMqttToken.isComplete();
log.info("MQTT connect"+(complete?"success":"fail"));
/** Subscribe to topics**/
for (String topic : topicList) {
log.info("Connection subscription topic:{}", topic);
//client.subscribe(topic, 0);
client.subscribe(topic, mqttMessageListener);
}
} catch (Exception e) {
e.printStackTrace();
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
- If the business also uses manual subscription, it also needs to inject MqttMessageListener on the subscription class and use it as a parameter in the subscription method. However, we need to change the method to non static, so we need to new the object when using this method. However, manual subscription is rarely used. Therefore, this step can be used with or without this step
import org.eclipse.paho.client.mqttv3.MqttClient;
import org.eclipse.paho.client.mqttv3.MqttException;
import org.eclipse.paho.client.mqttv3.MqttMessage;
import org.eclipse.paho.client.mqttv3.MqttTopic;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
/**
* @Author chy
* @Date
* @Description
*/
public class TopicOperation {
private static final Logger log = LoggerFactory.getLogger(TopicOperation.class);
//Inject MqttMessageListener
@Autowired
private MqttMessageListener mqttMessageListener;
/**
* Subscribe to topics
* @param topic Subject name
*/
public void subscribe(String topic) {
try {
MqttClient client = MqttPushClient.getClient();
if (client == null) {
return;
};
//client.subscribe(topic, 0);
//Used as a parameter in the subscription method
client.subscribe(topic, mqttMessageListener);
log.info("Subscribe to topics:{}",topic);
} catch (MqttException e) {
e.printStackTrace();
}
}
/**
* Publish theme
*
* @param topic
* @param pushMessage
*/
public static void publish(String topic, String pushMessage) {
log.info("SEND TO MQTT -- topic : {}, message : {}", topic, pushMessage);
MqttMessage message = new MqttMessage();
message.setQos(0);
// Non persistent
message.setRetained(false);
message.setPayload(pushMessage.getBytes());
MqttClient client = MqttPushClient.getClient();
if (client == null) {
return;
};
MqttTopic mTopic = client.getTopic(topic);
if (null == mTopic) {
log.error("Subject does not exist:{}",mTopic);
}
try {
mTopic.publish(message);
} catch (Exception e) {
log.error("mqtt Sending message exception:",e);
}
}
}
Open asynchronous threads through annotations and configure processing methods
- Start class: enable @ EnableAsync(proxyTargetClass=true)
@SpringBootApplication
@MapperScan(basePackages = "com.x.x.mapper")
@EnableTransactionManagement
@EnableAsync(proxyTargetClass=true )
public class XXApplication {
public static void main(String[] args) {
SpringApplication.run(XXApplication.class, args);
}
}
- Configuration class configuration thread pool parameters
@Slf4j
@Configuration
public class ExecutorConfig {
@Bean
public Executor asyncServiceExecutor() {
log.info("start asyncServiceExecutor");
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
//Configure the number of core threads
executor.setCorePoolSize(9);
//Configure maximum threads
executor.setMaxPoolSize(20);
//Configure queue size
executor.setQueueCapacity(200);
//Configure the name prefix of threads in the thread pool
executor.setThreadNamePrefix("sxd-async-service-");
// Set rejection policy: how to handle new tasks when the pool has reached max size
// CALLER_RUNS: the task is not executed in the new thread, but in the thread of the caller
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
//Perform initialization
executor.initialize();
return executor;
}
}
- The implementation method of MqttMessageListener messageArrived enables @ Async("asyncServiceExecutor")
@Slf4j
@Component
public class MqttMessageListener implements IMqttMessageListener {
@Resource
private BusinessService businessService;
@Autowired
private MqttTopic mqttTopic;
@Autowired
private ThreeCallmachineService threeCallmachineService;
@Autowired
private BusinessHallService businessHallService;
@Autowired
private BusinessMaterialService businessMaterialService;
@Autowired
private BusinessWaitService businessWaitService;
@Autowired
private AppointmentService appointmentService;
@Override
@Async("asyncServiceExecutor")
public void messageArrived(String topic, MqttMessage message) throws Exception {
String messageStr = new String(message.getPayload());
log.info("Received subject:" + topic + ";Information received:" + messageStr);
System.out.println("Thread Name:[" + Thread.currentThread().getName() + "]");
//Conduct business processing
}
}