ConsumeQueue for source code analysis

When sending a message, the data is displayed in the ConsumeQueue

Five messages are sent continuously. The messages are of variable length. First, all information is put into the Commitlog. Each message needs to be locked when it is put into the Commitlog to ensure sequential writing.

​

After the Commitlog is written successfully. The data is resynchronized to the consumequeue.

And the data is distributed one by one. This is a typical polling.

Queue Offset represents the number of messages in a queue

Logic Offset is Queue Offset*20, because the message length in each ConsumeQueue is 20

Physical Offset, which is the offset of each message in the Commitlog.

This design is very clever:

When searching for a message, you can directly calculate the global location of the index according to the message sequence number of the queue (for example, if the sequence number is 2, you know that the offset is 20), then directly read the index, and then find the message according to the global location of the message recorded in the index. The two operations are to find the index and the file where the message is located, These two searches are similar and can be abstracted into:

Because the length of each index file or message file is fixed, an orderly file array from small to large is maintained for each group of files. When searching for a file, you can directly obtain the serial number of the file in the array through calculation:

Sequence number of the file in the array = (global location - file name of the first file) / fixed file size

The time complexity of obtaining data in the array through sequence number is 0 (1), and the time complexity of secondary file search is: 0 (1) + 0 (1) = 0 (1). Therefore, the time complexity of finding data during consumption is also O(1).

Entry: reputmessageservice.doreput (independent thread)

DefaultMessageStore. start()

// org.apache.rocketmq.store.DefaultMessageStore#start
log.info("[SetReputOffset] maxPhysicalPosInLogicQueue={} clMinOffset={} clMaxOffset={} clConfirmedOffset={}",
         maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset(), this.commitLog.getMaxOffset(),
         this.commitLog.getConfirmOffset());
this.reputMessageService.setReputFromOffset(maxPhysicalPosInLogicQueue);
this.reputMessageService.start(); // here

ReputMessageService.run()

// org.apache.rocketmq.store.DefaultMessageStore.ReputMessageService#run
public void run() {
    DefaultMessageStore.log.info(this.getServiceName() + " service started");
    while (!this.isStopped()) {
        try {
            Thread.sleep(1);
            this.doReput(); // here
        } catch (Exception e) {
            DefaultMessageStore.log.warn(this.getServiceName() + " service has exception. ", e);
        }
    }
    DefaultMessageStore.log.info(this.getServiceName() + " service end");
}
// org.apache.rocketmq.store.DefaultMessageStore.ReputMessageService#doReput
private void doReput() {
    if (this.reputFromOffset < DefaultMessageStore.this.commitLog.getMinOffset()) {
        this.reputFromOffset = DefaultMessageStore.this.commitLog.getMinOffset();
    }
    for (boolean doNext = true; this.isCommitLogAvailable() && doNext; ) {
        if (DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable()
            && this.reputFromOffset >= DefaultMessageStore.this.getConfirmOffset()) {
            break;
        }
        //It mainly builds ConsumerQueue and Index
        //reputFromOffset: progress of building ConsumerQueue and Index
        SelectMappedBufferResult result = DefaultMessageStore.this.commitLog.getData(reputFromOffset);
        if (result != null) {
            ...
                DefaultMessageStore.this.doDispatch(dispatchRequest); // here
            ...
        }
    }
}
// org.apache.rocketmq.store.DefaultMessageStore.CommitLogDispatcherBuildConsumeQueue
class CommitLogDispatcherBuildConsumeQueue implements CommitLogDispatcher {

    @Override
    public void dispatch(DispatchRequest request) {
        final int tranType = MessageSysFlag.getTransactionValue(request.getSysFlag());
        switch (tranType) {
            case MessageSysFlag.TRANSACTION_NOT_TYPE:
            case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
                // Build a logical queue for the consumer
                DefaultMessageStore.this.putMessagePositionInfo(request); // here
                break;
            case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
            case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
                break;
        }
    }
}
// org.apache.rocketmq.store.DefaultMessageStore#putMessagePositionInfo
public void putMessagePositionInfo(DispatchRequest dispatchRequest) {
    ConsumeQueue cq = this.findConsumeQueue(dispatchRequest.getTopic(), dispatchRequest.getQueueId());
    cq.putMessagePositionInfoWrapper(dispatchRequest);
}

public void putMessagePositionInfoWrapper(DispatchRequest request) {
...
    // Build a logical queue for the consumer
    boolean result = this.putMessagePositionInfo(
    request.getCommitLogOffset(), request.getMsgSize(), 
    tagsCode, request.getConsumeQueueOffset());
}

// org.apache.rocketmq.store.ConsumeQueue#putMessagePositionInfo
private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode,
                                       final long cqOffset) {
    ...
	MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(expectLogicOffset); // here
    if (mappedFile != null) {
        return mappedFile.appendMessage(this.byteBufferIndex.array()); // here
    }
}
// org.apache.rocketmq.store.MappedFile#appendMessage(byte[])
public boolean appendMessage(final byte[] data) {
    int currentPos = this.wrotePosition.get();
    if ((currentPos + data.length) <= this.fileSize) {
        try {
            this.fileChannel.position(currentPos);
            this.fileChannel.write(ByteBuffer.wrap(data));
        } catch (Throwable e) {
            log.error("Error occurred when append message to mappedFile.", e);
        }
        this.wrotePosition.addAndGet(data.length);
        return true;
    }
    return false;
}

Asynchronous brush disk

// org.apache.rocketmq.store.DefaultMessageStore#DefaultMessageStore
public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, 
                            final BrokerStatsManager brokerStatsManager,
                            final MessageArrivingListener messageArrivingListener,
                            final BrokerConfig brokerConfig) throws IOException {
     this.messageArrivingListener = messageArrivingListener;
     this.brokerConfig = brokerConfig;
     this.messageStoreConfig = messageStoreConfig;
     this.brokerStatsManager = brokerStatsManager;
     this.allocateMappedFileService = new AllocateMappedFileService(this);
     if (messageStoreConfig.isEnableDLegerCommitLog()) {
         this.commitLog = new DLedgerCommitLog(this);
     } else {
         this.commitLog = new CommitLog(this);
     }
     this.consumeQueueTable = new ConcurrentHashMap<>(32);

     this.flushConsumeQueueService = new FlushConsumeQueueService(); // here
 }
// 
private void doFlush(int retryTimes) {
...
    if (0 == flushConsumeQueueLeastPages) {
        if (logicsMsgTimestamp > 0) {
            DefaultMessageStore.this.getStoreCheckpoint().setLogicsMsgTimestamp(logicsMsgTimestamp);
        }
        DefaultMessageStore.this.getStoreCheckpoint().flush(); // here
    }
}