catalogue
Implementation principle of redis blocking command
Disadvantages of delayed queues
Implementation principle of redis blocking command
blpop and brpop in redis can block the list, and the client connection will block when the list has no data.
Here you may have a question: redis itself is a single threaded service. If the blocking client keeps the link with the server, will it block the execution of other commands?
There are two cycles in the redis server:
1. IO cycles and timed events. In the IO cycle, redis completes the client connection response, command request processing and command processing result reply.
2. In the timing cycle, redis completes the detection of expired key s, etc.
The process of redis one-time connection processing includes several important steps: IO multiplexing, detecting socket status, socket event dispatching and request event processing. During the blpop command processing, redis will first look up the list corresponding to the key. If it exists, the pop will send the data response to the client. Otherwise, push the corresponding key to blocking_ In the keys data structure, the corresponding value is the blocked client. When the next push command is issued, the server checks blocking_ Whether there is a corresponding key in keys. If so, add the key to ready_keys linked list, insert value into the linked list and respond to the client.
After processing the client request in each event loop, the server will traverse ready_keys linked list, and from blocking_ Find the corresponding client in the keys linked list and respond. The whole process will not block the execution of the event loop. Therefore, generally speaking, the redis server is through ready_keys and blocking_keys two linked lists and event loops to handle blocking events.
Disadvantages of delayed queues
The delayed message queue implemented by Redis also has the problems of data persistence and message reliability
No retry mechanism - there is no retry mechanism for exceptions in message processing. These need to be implemented by yourself, including the implementation of retry times
There is no ACK mechanism - for example, when the message is obtained and deleted, the client crashes while processing the message, and the messages being processed will be lost. MQ needs to explicitly return a value to MQ before it considers that the message is correctly consumed
If you require high message reliability, MQ is recommended
Basic implementation
Related interface
import java.util.Optional;
public interface IQueue<E> {
boolean add(E item);
Optional<E> get();
Optional<E> get(int timeout);
long size();
}abstract class
import com.google.gson.Gson;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.logging.log4j.util.Strings;
import org.slf4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import redis.clients.jedis.JedisCluster;
import javax.annotation.PostConstruct;
import java.util.List;
import java.util.Optional;
/**
* redis The queue abstraction class abstracts the code of the common part of the redis queue
* To implement the class, you need to:
* 1.When initializing, define QUEUE_NAME to specify the queue name
* 2.Set log
* 3.Implement the convert method, and use Gson to convert the read content into a specific object
* @param <E>
*/
public abstract class AbstractRedisQueue<E> implements IQueue<E> {
private static final int DEFAULT_TIMEOUT = 5;
String QUEUE_NAME;
Logger log;
@Autowired
private JedisCluster jedisCluster;
@PostConstruct
void init(){
setProperty();
if (Strings.isBlank(QUEUE_NAME)) {
throw new RuntimeException("queue name can't be empty!");
}
if (log == null) {
throw new RuntimeException("not set logger!");
}
log.info("Currently used redis Queue is:{}", QUEUE_NAME);
}
@Override
public boolean add(E item) {
try {
log.debug("add to redis queue. queueName = {}, content = {}", QUEUE_NAME, item);
long insertResult = jedisCluster.lpush(QUEUE_NAME, toString(item));
log.debug("add to redis queue, result = {}", insertResult);
return true;
}catch (Exception e){
log.error("add to redisQueue exception. queueName = {}, error = {}", QUEUE_NAME, e.getMessage(), e);
}
return false;
}
public Optional<E> get(){
return get(DEFAULT_TIMEOUT);
}
@Override
public Optional<E> get(int timeout) {
List<String> pairResult = jedisCluster.brpop(timeout, QUEUE_NAME);
log.debug("brpop from redis queue,result = {}", pairResult);
if (CollectionUtils.isEmpty(pairResult)) {
log.info("queue is empty!");
return Optional.empty();
}
if (CollectionUtils.size(pairResult) != 2 || !StringUtils.equals(pairResult.get(0), QUEUE_NAME)) {
log.error("brpop from redis queue error, result = {}", pairResult);
return Optional.empty();
}
String content = pairResult.get(1);
log.info("brpop from redis queue. queueName = {}, content = {}", QUEUE_NAME, content);
return Optional.of(convert(content));
}
String toString(E item){
return new Gson().toJson(item);
}
@Override
public long size() {
return jedisCluster.llen(QUEUE_NAME);
}
/**
* Set queue name and log
*/
abstract void setProperty();
abstract E convert(String str);
}Implementation class
import com.google.gson.Gson;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class OrderRedisQueue extends AbstractRedisQueue<MyItem> {
@Override
public void setProperty(){
QUEUE_NAME = "myQueue";
super.log = log;
}
@Override
MyItem convert(String str) {
return new Gson().fromJson(str, MyItem.class);
}
}
Call class
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.PostConstruct;
import javax.inject.Singleton;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.MDC;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
@Slf4j
@Component
@Lazy(false)
@Singleton
public class OrderConsumerTask {
private ExecutorService executorService;
private final AtomicBoolean consumerStopFlag = new AtomicBoolean(false);
@Autowired
private OrderRedisQueue orderRedisQueue;
private List<Future> futures;
@PostConstruct
public void startUpConsumerTask() {
log.info("Start the order log store request consumption thread");
executorService = Executors.newSingleThreadExecutor();
futures = new ArrayList<>(1);
for (int i = 0; i < 50; ++i) {
futures.add(executorService.submit(new OrderConsumer()));
}
log.info("End of order log storage request consumption thread");
}
// To stop consuming threads, and try to end the service after thread consumption
public void stopConsumerService() {
log.info("Stop the store request consumer thread");
this.consumerStopFlag.set(true);
// Submit is convenient for exception handling. If your task throws checked or unchecked exception s, and you want external callers to perceive these exceptions and handle them in time, you need to use submit to catch the exceptions thrown by Future.get
futures.forEach(future -> {
try {
future.get();
} catch (ExecutionException | InterruptedException e) {
log.error(e.getMessage(), e);
}
});
futures.clear();
executorService.shutdown();
log.info("Stop store request consumer thread complete");
}
class OrderConsumer implements Runnable {
@Override
public void run() {
while (!consumerStopFlag.get()) {
try {
MDC.put("SessionId", "Storage processing thread" + UUID.randomUUID().toString().substring(0, 5));
Optional<MyItem> optional = orderRedisQueue.get();
if (!optional.isPresent()) {
continue;
}
MyItem myItem = optional.get();
// Business processing
} catch (Throwable e) {
log.error(e.getMessage(), e);
} finally {
MDC.clear();
}
}
log.info("Store request consumer thread exit.");
}
}
}
reference resources
Principle of blpop in redis_ wszylh's blog - CSDN blog
Redis based implementation of simple message queue and delayed message queue - Zhihu